From e4a9f98a0c1adbddae90f6d9eb38f33dea5a5b2d Mon Sep 17 00:00:00 2001 From: Bogdan Date: Sat, 8 Apr 2023 02:17:49 +0300 Subject: [PATCH] animebytes: refactor (#14230) * animebytes: use `.Value` * animebytes: refactor MST * animebytes: add freeleech only/exclude hentai options * animebytes: search by year only when search type is anime * animebytes: add S01 only to anime * animebytes: refactor properties --- src/Jackett.Common/Indexers/AnimeBytes.cs | 282 ++++++++++++------ .../Bespoke/ConfigurationDataAnimeBytes.cs | 8 +- 2 files changed, 191 insertions(+), 99 deletions(-) diff --git a/src/Jackett.Common/Indexers/AnimeBytes.cs b/src/Jackett.Common/Indexers/AnimeBytes.cs index 528702f0b..1963e27de 100644 --- a/src/Jackett.Common/Indexers/AnimeBytes.cs +++ b/src/Jackett.Common/Indexers/AnimeBytes.cs @@ -13,7 +13,6 @@ using Jackett.Common.Models; using Jackett.Common.Models.IndexerConfig.Bespoke; using Jackett.Common.Services.Interfaces; using Jackett.Common.Utils; -using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NLog; using WebClient = Jackett.Common.Utils.Clients.WebClient; @@ -43,6 +42,12 @@ namespace Jackett.Common.Indexers private static Regex YearRegex => new Regex(@"\b((?:19|20)\d{2})$", RegexOptions.Compiled); + private readonly HashSet _excludedProperties = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "Freeleech", + "Hentai" + }; + private ConfigurationDataAnimeBytes ConfigData => (ConfigurationDataAnimeBytes)configData; public AnimeBytes(IIndexerConfigurationService configService, WebClient client, Logger l, @@ -126,7 +131,9 @@ namespace Jackett.Common.Indexers releases.AddRange(await GetResults(query, "anime", StripEpisodeNumber(query.SanitizedSearchTerm.Trim()))); if (ContainsMusicCategories(query.Categories)) + { releases.AddRange(await GetResults(query, "music", query.SanitizedSearchTerm.Trim())); + } return releases .OrderByDescending(o => o.PublishDate) @@ -189,7 +196,7 @@ namespace Jackett.Common.Indexers { "searchstr", searchTerm } }; - if (ConfigData.SearchByYear.Value) + if (ConfigData.SearchByYear.Value && searchType == "anime") { var searchYear = ParseYearFromSearchTerm(query.SanitizedSearchTerm.Trim()); @@ -206,6 +213,16 @@ namespace Jackett.Common.Indexers queryCats.ForEach(cat => parameters.Set(cat, "1")); } + if (ConfigData.FreeleechOnly.Value) + { + parameters.Set("freeleech", "1"); + } + + if (ConfigData.ExcludeHentai.Value && searchType == "anime") + { + parameters.Set("hentai", "0"); + } + var searchUrl = ScrapeUrl + "?" + parameters.GetQueryString(); // Check cache first so we don't query the server for each episode when searching for each episode in a series. @@ -224,92 +241,150 @@ namespace Jackett.Common.Indexers // Get the content from the tracker var response = await RequestWithCookiesAndRetryAsync(searchUrl); + if (!response.ContentString.StartsWith("{")) // not JSON => error + { throw new ExceptionWithConfigData("Unexpected response (not JSON)", ConfigData); + } - var json = JsonConvert.DeserializeObject(response.ContentString); - - // Parse try { - if (json["error"] != null) - throw new Exception(json["error"].ToString()); + var json = JToken.Parse(response.ContentString); - var matches = (long)json["Matches"]; + if (json.Value("error") != null) + { + throw new Exception(json.Value("error")); + } - if (matches == 0) + if (json.Value("Matches") == 0) { return releases; } - var groups = (JArray)json.Groups; - - foreach (var group in groups) + foreach (var group in json.Value("Groups")) { - var synonyms = new List(); - var posterStr = (string)group["Image"]; - var poster = (string.IsNullOrWhiteSpace(posterStr) ? null : new Uri(posterStr)); - var year = (int)group["Year"]; - var groupName = (string)group["GroupName"]; - var seriesName = (string)group["SeriesName"]; - var mainTitle = WebUtility.HtmlDecode((string)group["FullName"]); + var categoryName = group.Value("CategoryName"); + var description = group.Value("Description"); + var year = group.Value("Year"); + var posterStr = group.Value("Image"); + var poster = posterStr.IsNotNullOrWhiteSpace() ? new Uri(posterStr) : null; + var groupName = group.Value("GroupName"); + var seriesName = group.Value("SeriesName"); + var mainTitle = WebUtility.HtmlDecode(group.Value("FullName")); if (seriesName.IsNotNullOrWhiteSpace()) - mainTitle = seriesName; - - synonyms.Add(mainTitle); - - if (group["Synonymns"].HasValues) { - if (group["Synonymns"] is JArray) + mainTitle = seriesName; + } + + var synonyms = new HashSet { mainTitle }; + + if (group.Value("Synonymns").HasValues) + { + if (group.Value("Synonymns") is JArray) { - var allSyonyms = group["Synonymns"].ToObject>(); + var allSyonyms = group.Value("Synonymns").ToObject>(); if (AddJapaneseTitle && allSyonyms.Count >= 1) - synonyms.Add(allSyonyms[0]); - if (AddRomajiTitle && allSyonyms.Count >= 2) - synonyms.Add(allSyonyms[1]); - if (AddAlternativeTitles && allSyonyms.Count >= 3) - synonyms.AddRange(allSyonyms[2].Split(',').Select(t => t.Trim())); - } - else - { - var allSynonyms = group["Synonymns"].ToObject>(); - - if (AddJapaneseTitle && allSynonyms.ContainsKey(0)) - synonyms.Add(allSynonyms[0]); - if (AddRomajiTitle && allSynonyms.ContainsKey(1)) - synonyms.Add(allSynonyms[1]); - if (AddAlternativeTitles && allSynonyms.ContainsKey(2)) { - synonyms.AddRange(allSynonyms[2].Split(',').Select(t => t.Trim())); + synonyms.Add(allSyonyms[0]); + } + + if (AddRomajiTitle && allSyonyms.Count >= 2) + { + synonyms.Add(allSyonyms[1]); + } + + if (AddAlternativeTitles && allSyonyms.Count >= 3) + { + synonyms.UnionWith(allSyonyms[2].Split(',').Select(t => t.Trim())); + } + } + else if (group.Value("Synonymns") is JObject) + { + var allSynonyms = group.Value("Synonymns").ToObject>(); + + if (AddJapaneseTitle && allSynonyms.TryGetValue(0, out var japaneseTitle)) + { + synonyms.Add(japaneseTitle.Trim()); + } + + if (AddRomajiTitle && allSynonyms.TryGetValue(1, out var romajiTitle)) + { + synonyms.Add(romajiTitle.Trim()); + } + + if (AddAlternativeTitles && allSynonyms.TryGetValue(2, out var alternativeTitles)) + { + synonyms.UnionWith(alternativeTitles.Split(',').Select(t => t.Trim())); } } } List category = null; - var categoryName = (string)group["CategoryName"]; - var description = (string)group["Description"]; - - foreach (var torrent in group["Torrents"]) + foreach (var torrent in group.Value("Torrents")) { - var releaseInfo = "S01"; + // Skip non-freeleech results when freeleech only is set + if (ConfigData.FreeleechOnly.Value && torrent.Value("RawDownMultiplier") != 0) + { + continue; + } + + var torrentId = torrent.Value("ID"); + var link = torrent.Value("Link"); + var linkUri = new Uri(link); + var publishDate = DateTime.ParseExact(torrent.Value("UploadTime"), "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal); + var details = new Uri(SiteLink + "torrent/" + torrentId + "/group"); + var size = torrent.Value("Size"); + var snatched = torrent.Value("Snatched"); + var seeders = torrent.Value("Seeders"); + var leechers = torrent.Value("Leechers"); + var peers = seeders + leechers; + var fileCount = torrent.Value("FileCount"); + var rawDownMultiplier = torrent.Value("RawDownMultiplier"); + var rawUpMultiplier = torrent.Value("RawUpMultiplier"); + + // MST with additional 5 hours per GB + var minimumSeedTime = 259200 + (int)(size / (int)Math.Pow(1024, 3) * 18000); + + var properties = WebUtility.HtmlDecode(torrent.Value("Property")) + .Split('|') + .Select(t => t.Trim()) + .Where(p => p.IsNotNullOrWhiteSpace()) + .ToList(); + + properties.RemoveAll(p => _excludedProperties.Any(p.Contains)); + + if (!AllowRaws && properties.ContainsIgnoreCase("RAW")) + { + continue; + } + + var releaseInfo = categoryName == "Anime" ? "S01" : ""; + var editionTitle = torrent.Value("EditionData")?.Value("EditionTitle"); + string episode = null; int? season = null; - var editionTitle = (string)torrent["EditionData"]["EditionTitle"]; - if (!string.IsNullOrWhiteSpace(editionTitle)) + + if (editionTitle.IsNotNullOrWhiteSpace()) + { releaseInfo = WebUtility.HtmlDecode(editionTitle); + } var seasonRegEx = new Regex(@"Season (\d+)", RegexOptions.Compiled); var seasonRegExMatch = seasonRegEx.Match(releaseInfo); if (seasonRegExMatch.Success) + { season = ParseUtil.CoerceInt(seasonRegExMatch.Groups[1].Value); + } var episodeRegEx = new Regex(@"Episode (\d+)", RegexOptions.Compiled); var episodeRegExMatch = episodeRegEx.Match(releaseInfo); if (episodeRegExMatch.Success) + { episode = episodeRegExMatch.Groups[1].Value; + } releaseInfo = releaseInfo.Replace("Episode ", ""); releaseInfo = releaseInfo.Replace("Season ", "S"); @@ -323,100 +398,112 @@ namespace Jackett.Common.Indexers if (FilterSeasonEpisode) { if (query.Season != 0 && season != null && season != query.Season) // skip if season doesn't match + { continue; - if (query.Episode != null && episode != null && episode != query.Episode) // skip if episode doesn't match - continue; - } - var torrentId = (long)torrent["ID"]; - var property = ((string)torrent["Property"]).Replace(" | Freeleech", ""); - var link = (string)torrent["Link"]; - var linkUri = new Uri(link); - var uploadTimeString = (string)torrent["UploadTime"]; - var uploadTime = DateTime.ParseExact(uploadTimeString, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture); - var publishDate = DateTime.SpecifyKind(uploadTime, DateTimeKind.Utc).ToLocalTime(); - var details = new Uri(SiteLink + "torrent/" + torrentId + "/group"); - var size = (long)torrent["Size"]; - var snatched = (long)torrent["Snatched"]; - var seeders = (int)torrent["Seeders"]; - var leechers = (int)torrent["Leechers"]; - var fileCount = (long)torrent["FileCount"]; - var peers = seeders + leechers; + } - var rawDownMultiplier = (int?)torrent["RawDownMultiplier"] ?? 0; - var rawUpMultiplier = (int?)torrent["RawUpMultiplier"] ?? 0; + if (query.Episode != null && episode != null && episode != query.Episode) // skip if episode doesn't match + { + continue; + } + } if (searchType == "anime") { if (groupName == "TV Series" || groupName == "OVA") + { category = new List { TorznabCatType.TVAnime.ID }; + } // Ignore these categories as they'll cause hell with the matcher // TV Special, OVA, ONA, DVD Special, BD Special if (groupName == "Movie" || groupName == "Live Action Movie") + { category = new List { TorznabCatType.Movies.ID }; + } if (categoryName == "Manga" || categoryName == "Oneshot" || categoryName == "Anthology" || categoryName == "Manhwa" || categoryName == "Manhua" || categoryName == "Light Novel") + { category = new List { TorznabCatType.BooksComics.ID }; + } if (categoryName == "Novel" || categoryName == "Artbook") + { category = new List { TorznabCatType.BooksComics.ID }; + } if (categoryName == "Game" || categoryName == "Visual Novel") { - if (property.Contains(" PSP ")) + if (properties.Contains("PSP")) + { category = new List { TorznabCatType.ConsolePSP.ID }; - if (property.Contains("PSX") || property.Contains(" NES ") || property.Contains(" Switch ")) + } + + if (properties.Contains("PS3")) + { + category = new List { TorznabCatType.ConsolePS3.ID }; + } + + if (properties.Contains("PS Vita")) + { + category = new List { TorznabCatType.ConsolePSVita.ID }; + } + + if (properties.Contains("3DS")) + { + category = new List { TorznabCatType.Console3DS.ID }; + } + + if (properties.Contains("NDS")) + { + category = new List { TorznabCatType.ConsoleNDS.ID }; + } + + if (properties.Contains("PSX") || properties.Contains("PS2") || properties.Contains("SNES") || properties.Contains("NES") || properties.Contains("GBA") || properties.Contains("Switch")) + { category = new List { TorznabCatType.ConsoleOther.ID }; - if (property.Contains(" PC ")) + } + + if (properties.Contains("PC")) + { category = new List { TorznabCatType.PCGames.ID }; + } } } else if (searchType == "music") { if (categoryName == "Single" || categoryName == "EP" || categoryName == "Album" || categoryName == "Compilation" || categoryName == "Soundtrack" || categoryName == "Remix CD" || categoryName == "PV" || categoryName == "Live Album" || categoryName == "Image CD" || categoryName == "Drama CD" || categoryName == "Vocal CD") { - if (property.Contains(" Lossless ")) + if (properties.Contains("Lossless")) + { category = new List { TorznabCatType.AudioLossless.ID }; - else if (property.Contains("MP3")) + } + else if (properties.Contains("MP3")) + { category = new List { TorznabCatType.AudioMP3.ID }; + } else + { category = new List { TorznabCatType.AudioOther.ID }; + } } } // We don't actually have a release name >.> so try to create one - var releaseTags = property.Split("|".ToCharArray(), StringSplitOptions.RemoveEmptyEntries).ToList(); - for (var i = releaseTags.Count - 1; i >= 0; i--) - { - releaseTags[i] = releaseTags[i].Trim(); - if (string.IsNullOrWhiteSpace(releaseTags[i])) - releaseTags.RemoveAt(i); - } + var releaseGroup = properties.LastOrDefault(); - var releaseGroup = releaseTags.LastOrDefault(); - if (releaseGroup != null && releaseGroup.Contains("(") && releaseGroup.Contains(")")) + if (releaseGroup.IsNotNullOrWhiteSpace() && releaseGroup.Contains("(") && releaseGroup.Contains(")")) { - // Skip raws if set - if (releaseGroup.ToLowerInvariant().StartsWith("raw") && !AllowRaws) - { - continue; - } - var start = releaseGroup.IndexOf("(", StringComparison.Ordinal); - releaseGroup = "[" + releaseGroup.Substring(start + 1, (releaseGroup.IndexOf(")", StringComparison.Ordinal) - 1) - start) + "] "; + releaseGroup = "[" + releaseGroup.Substring(start + 1, releaseGroup.IndexOf(")", StringComparison.Ordinal) - 1 - start) + "] "; } else { releaseGroup = string.Empty; } - if (!AllowRaws && releaseTags.Contains("raw", StringComparer.InvariantCultureIgnoreCase)) - continue; - var infoString = releaseTags.Aggregate("", (prev, cur) => prev + "[" + cur + "]"); - var minimumSeedTime = 259200; - // Additional 5 hours per GB - minimumSeedTime += (int)((size / 1000000000) * 18000); + var infoString = properties.Select(p => "[" + p + "]").Join(string.Empty); foreach (var title in synonyms) { @@ -450,11 +537,12 @@ namespace Jackett.Common.Indexers releases.Add(release); } - if (AddFileNameTitles && (int)torrent["FileCount"] == 1) + if (AddFileNameTitles && fileCount == 1) { - var releaseTitle = Path.GetFileNameWithoutExtension((string)torrent["FileList"][0]["filename"]); + var releaseTitle = Path.GetFileNameWithoutExtension(torrent.Value("FileList")?.First().Value("filename")); var guid = new Uri(details + "&nh=" + StringUtil.Hash(releaseTitle)); + var release = new ReleaseInfo { MinimumRatio = 1, diff --git a/src/Jackett.Common/Models/IndexerConfig/Bespoke/ConfigurationDataAnimeBytes.cs b/src/Jackett.Common/Models/IndexerConfig/Bespoke/ConfigurationDataAnimeBytes.cs index 7ec2eaad7..4f96dac0e 100644 --- a/src/Jackett.Common/Models/IndexerConfig/Bespoke/ConfigurationDataAnimeBytes.cs +++ b/src/Jackett.Common/Models/IndexerConfig/Bespoke/ConfigurationDataAnimeBytes.cs @@ -5,8 +5,10 @@ namespace Jackett.Common.Models.IndexerConfig.Bespoke [ExcludeFromCodeCoverage] internal class ConfigurationDataAnimeBytes : ConfigurationDataUserPasskey { - public BoolConfigurationItem SearchByYear { get; private set; } + public BoolConfigurationItem FreeleechOnly { get; private set; } + public BoolConfigurationItem ExcludeHentai { get; private set; } public BoolConfigurationItem IncludeRaw { get; private set; } + public BoolConfigurationItem SearchByYear { get; private set; } //public DisplayItem DateWarning { get; private set; } public BoolConfigurationItem PadEpisode { get; private set; } public BoolConfigurationItem AddJapaneseTitle { get; private set; } @@ -18,8 +20,10 @@ namespace Jackett.Common.Models.IndexerConfig.Bespoke public ConfigurationDataAnimeBytes(string instructionMessageOptional = null) : base(instructionMessageOptional) { + FreeleechOnly = new BoolConfigurationItem("Search freeleech only") { Value = false }; + ExcludeHentai = new BoolConfigurationItem("Exclude Hentai from results") { Value = false }; + IncludeRaw = new BoolConfigurationItem("Include RAW in results") { Value = false }; SearchByYear = new BoolConfigurationItem("Search by year as a different argument in the request") { Value = false }; - IncludeRaw = new BoolConfigurationItem("IncludeRaw") { Value = false }; //DateWarning = new DisplayItem("This tracker does not supply upload dates so they are based off year of release.") { Name = "DateWarning" }; PadEpisode = new BoolConfigurationItem("Pad episode number for Sonarr compatability") { Value = false }; AddJapaneseTitle = new BoolConfigurationItem("Add releases for Japanese Title") { Value = false };