diff --git a/src/Jackett.Common/Definitions/knaben.yml b/src/Jackett.Common/Definitions/knaben.yml deleted file mode 100644 index 1afa097d8..000000000 --- a/src/Jackett.Common/Definitions/knaben.yml +++ /dev/null @@ -1,160 +0,0 @@ ---- -id: knaben -name: Knaben -description: "Knaben is a Public torrent meta-search engine" -language: en-US -type: public -encoding: UTF-8 -links: - - https://knaben.eu/ - -caps: - categorymappings: - - {id: 1000000, cat: Audio, desc: "Audio"} - - {id: 1001000, cat: Audio/MP3, desc: "MP3"} - - {id: 1002000, cat: Audio/Lossless, desc: "Lossless"} - - {id: 1003000, cat: Audio/Audiobook, desc: "Audiobook"} - - {id: 1004000, cat: Audio/Video, desc: "Audio Video"} - - {id: 1005000, cat: Audio/Other, desc: "Radio"} - - {id: 1006000, cat: Audio/Other, desc: "Audio Other"} - - {id: 2000000, cat: TV, desc: "TV"} - - {id: 2001000, cat: TV/HD, desc: "TV HD"} - - {id: 2002000, cat: TV/SD, desc: "TV SD"} - - {id: 2003000, cat: TV/UHD, desc: "TV UHD"} - - {id: 2004000, cat: TV/Documentary, desc: "Documentary"} - - {id: 2005000, cat: TV/Foreign, desc: "TV Foreign"} - - {id: 2006000, cat: TV/Sport, desc: "Sport"} - - {id: 2007000, cat: TV/Other, desc: "Cartoon"} - - {id: 2008000, cat: TV/Other, desc: "TV Other"} - - {id: 3000000, cat: Movies, desc: "Movies"} - - {id: 3001000, cat: Movies/HD, desc: "Movies HD"} - - {id: 3002000, cat: Movies/SD, desc: "Movies SD"} - - {id: 3003000, cat: Movies/UHD, desc: "Movies UHD"} - - {id: 3004000, cat: Movies/DVD, desc: "Movies DVD"} - - {id: 3005000, cat: Movies/Foreign, desc: "Movies Foreign"} - - {id: 3006000, cat: Movies/Foreign, desc: "Movies Bollywood"} - - {id: 3007000, cat: Movies/3D, desc: "Movies 3D"} - - {id: 3008000, cat: Movies/Other, desc: "Movies Other"} - - {id: 4000000, cat: PC, desc: "PC"} - - {id: 4001000, cat: PC/Games, desc: "Games"} - - {id: 4002000, cat: PC/0day, desc: "Software"} - - {id: 4003000, cat: PC/Mac, desc: "Mac"} - - {id: 4004000, cat: PC/ISO, desc: "Unix"} - - {id: 5000000, cat: XXX, desc: "XXX"} - - {id: 5001000, cat: XXX/x264, desc: "XXX Video"} - - {id: 5002000, cat: XXX/ImageSet, desc: "XXX ImageSet"} - - {id: 5003000, cat: XXX/Other, desc: "XXX Games"} - - {id: 5004000, cat: XXX/Other, desc: "XXX Hentai"} - - {id: 5005000, cat: XXX/Other, desc: "XXX Other"} - - {id: 6000000, cat: TV/Anime, desc: "Anime"} - - {id: 6001000, cat: TV/Anime, desc: "Anime Subbed"} - - {id: 6002000, cat: TV/Anime, desc: "Anime Dubbed"} - - {id: 6003000, cat: TV/Anime, desc: "Anime Dual audio"} - - {id: 6004000, cat: TV/Anime, desc: "Anime Raw"} - - {id: 6005000, cat: Audio/Video, desc: "Music Video"} - - {id: 6006000, cat: Books/Other, desc: "Literature"} - - {id: 6007000, cat: Audio/Other, desc: "Music"} - - {id: 6008000, cat: TV/Anime, desc: "Anime non-english translated"} - - {id: 7000000, cat: Console, desc: "Console"} - - {id: 7001000, cat: Console/PS4, desc: "PS4"} - - {id: 7002000, cat: Console/PS3, desc: "PS3"} - - {id: 7003000, cat: Console/PS3, desc: "PS2"} - - {id: 7004000, cat: Console/PS3, desc: "PS1"} - - {id: 7005000, cat: Console/PS Vita, desc: "PS Vita"} - - {id: 7006000, cat: Console/PSP, desc: "PSP"} - - {id: 7007000, cat: Console/XBox 360, desc: "Xbox 360"} - - {id: 7008000, cat: Console/XBox, desc: "Xbox"} - - {id: 7009000, cat: Console/NDS, desc: "Switch"} - - {id: 7010000, cat: Console/NDS, desc: "NDS"} - - {id: 7011000, cat: Console/Wii, desc: "Wii"} - - {id: 7012000, cat: Console/WiiU, desc: "WiiU"} - - {id: 7013000, cat: Console/3DS, desc: "3DS"} - - {id: 7014000, cat: Console/Wii, desc: "GameCube"} - - {id: 7015000, cat: Console/Other, desc: "Other"} - - {id: 8000000, cat: PC/Mobile-Other, desc: "Mobile"} - - {id: 8001000, cat: PC/Mobile-Android, desc: "Android"} - - {id: 8002000, cat: PC/Mobile-iOS, desc: "IOS"} - - {id: 8003000, cat: PC/Mobile-Other, desc: "PC Other"} - - {id: 9000000, cat: Books, desc: "Books"} - - {id: 9001000, cat: Books/EBook, desc: "EBooks"} - - {id: 9002000, cat: Books/Comics, desc: "Comics"} - - {id: 9003000, cat: Books/Mags, desc: "Magazines"} - - {id: 9004000, cat: Books/Technical, desc: "Technical"} - - {id: 9005000, cat: Books/Other, desc: "Books Other"} - - {id: 10000000, cat: Other, desc: "Other"} - - {id: 10001000, cat: Other/Misc, desc: "Other Misc"} - - modes: - search: [q] - tv-search: [q, season, ep] - movie-search: [q] - music-search: [q] - book-search: [q] - -settings: - - name: sort - type: select - label: Sort requested from site - default: date - options: - date: created - seeders: seeders - bytes: size - title: title - - name: type - type: select - label: Order requested from site - default: "+" - options: - "+": desc - "-": asc - -search: - paths: - - path: "{{ if .Keywords }}search/{{ .Keywords }}{{ else }}browse{{ end }}/0/1/{{ .Config.type }}{{ .Config.sort }}" - - rows: - selector: tr[title^="Cached "], tr[title^="Checked "] - - fields: - category: - selector: a[href^="/browse/"]:last-child - attribute: href - filters: - - name: split - args: ["/", 2] - # replace subcat with cat: 1001069 > 1001000 - - name: re_replace - args: ["(\\d{3})$", "000"] - title: - selector: td:nth-child(2) a - details: - selector: td:last-child a - attribute: href - download: - selector: td:nth-child(2) a - attribute: href - description: - selector: td:last-child a - filters: - - name: prepend - args: "Source: " - date: - selector: td:nth-child(4) - attribute: title - filters: - - name: append - args: " +01:00" # CET - - name: dateparse - args: "yyyy-MM-dd HH:mm:ss zzz" - size: - selector: td:nth-child(3) - seeders: - selector: td:nth-child(5) - leechers: - selector: td:nth-child(6) - downloadvolumefactor: - text: 0 - uploadvolumefactor: - text: 1 -# engine n/a diff --git a/src/Jackett.Common/Indexers/Definitions/Knaben.cs b/src/Jackett.Common/Indexers/Definitions/Knaben.cs new file mode 100644 index 000000000..c5847a8a8 --- /dev/null +++ b/src/Jackett.Common/Indexers/Definitions/Knaben.cs @@ -0,0 +1,310 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Jackett.Common.Extensions; +using Jackett.Common.Models; +using Jackett.Common.Models.IndexerConfig; +using Jackett.Common.Services.Interfaces; +using Jackett.Common.Utils.Clients; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NLog; +using WebClient = Jackett.Common.Utils.Clients.WebClient; + +namespace Jackett.Common.Indexers.Definitions +{ + [ExcludeFromCodeCoverage] + public class Knaben : IndexerBase + { + public override string Id => "knaben"; + public override string Name => "Knaben"; + public override string Description => "Knaben is a Public torrent meta-search engine"; + public override string SiteLink { get; protected set; } = "https://knaben.eu/"; + public override string Language => "en-US"; + public override string Type => "public"; + + public override TorznabCapabilities TorznabCaps => SetCapabilities(); + + public Knaben(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps, ICacheService cs) + : base(configService: configService, + client: wc, + logger: l, + p: ps, + cacheService: cs, + configData: new ConfigurationData()) + { + } + + private static TorznabCapabilities SetCapabilities() + { + var caps = new TorznabCapabilities + { + TvSearchParams = new List + { + TvSearchParam.Q, TvSearchParam.Season, TvSearchParam.Ep + }, + MovieSearchParams = new List + { + MovieSearchParam.Q + }, + MusicSearchParams = new List + { + MusicSearchParam.Q + }, + BookSearchParams = new List + { + BookSearchParam.Q + } + }; + + caps.Categories.AddCategoryMapping(1000000, TorznabCatType.Audio,"Audio"); + caps.Categories.AddCategoryMapping(1001000, TorznabCatType.AudioMP3, "MP3"); + caps.Categories.AddCategoryMapping(1002000, TorznabCatType.AudioLossless, "Lossless"); + caps.Categories.AddCategoryMapping(1003000, TorznabCatType.AudioAudiobook, "Audiobook"); + caps.Categories.AddCategoryMapping(1004000, TorznabCatType.AudioVideo, "Audio Video"); + caps.Categories.AddCategoryMapping(1005000, TorznabCatType.AudioOther, "Radio"); + caps.Categories.AddCategoryMapping(1006000, TorznabCatType.AudioOther, "Audio Other"); + caps.Categories.AddCategoryMapping(2000000, TorznabCatType.TV, "TV"); + caps.Categories.AddCategoryMapping(2001000, TorznabCatType.TVHD, "TV HD"); + caps.Categories.AddCategoryMapping(2002000, TorznabCatType.TVSD, "TV SD"); + caps.Categories.AddCategoryMapping(2003000, TorznabCatType.TVUHD, "TV UHD"); + caps.Categories.AddCategoryMapping(2004000, TorznabCatType.TVDocumentary, "Documentary"); + caps.Categories.AddCategoryMapping(2005000, TorznabCatType.TVForeign, "TV Foreign"); + caps.Categories.AddCategoryMapping(2006000, TorznabCatType.TVSport, "Sport"); + caps.Categories.AddCategoryMapping(2007000, TorznabCatType.TVOther, "Cartoon"); + caps.Categories.AddCategoryMapping(2008000, TorznabCatType.TVOther, "TV Other"); + caps.Categories.AddCategoryMapping(3000000, TorznabCatType.Movies, "Movies"); + caps.Categories.AddCategoryMapping(3001000, TorznabCatType.MoviesHD, "Movies HD"); + caps.Categories.AddCategoryMapping(3002000, TorznabCatType.MoviesSD, "Movies SD"); + caps.Categories.AddCategoryMapping(3003000, TorznabCatType.MoviesUHD, "Movies UHD"); + caps.Categories.AddCategoryMapping(3004000, TorznabCatType.MoviesDVD, "Movies DVD"); + caps.Categories.AddCategoryMapping(3005000, TorznabCatType.MoviesForeign, "Movies Foreign"); + caps.Categories.AddCategoryMapping(3006000, TorznabCatType.MoviesForeign, "Movies Bollywood"); + caps.Categories.AddCategoryMapping(3007000, TorznabCatType.Movies3D, "Movies 3D"); + caps.Categories.AddCategoryMapping(3008000, TorznabCatType.MoviesOther, "Movies Other"); + caps.Categories.AddCategoryMapping(4000000, TorznabCatType.PC, "PC"); + caps.Categories.AddCategoryMapping(4001000, TorznabCatType.PCGames, "Games"); + caps.Categories.AddCategoryMapping(4002000, TorznabCatType.PC0day, "Software"); + caps.Categories.AddCategoryMapping(4003000, TorznabCatType.PCMac, "Mac"); + caps.Categories.AddCategoryMapping(4004000, TorznabCatType.PCISO, "Unix"); + caps.Categories.AddCategoryMapping(5000000, TorznabCatType.XXX, "XXX"); + caps.Categories.AddCategoryMapping(5001000, TorznabCatType.XXXx264, "XXX Video"); + caps.Categories.AddCategoryMapping(5002000, TorznabCatType.XXXImageSet, "XXX ImageSet"); + caps.Categories.AddCategoryMapping(5003000, TorznabCatType.XXXOther, "XXX Games"); + caps.Categories.AddCategoryMapping(5004000, TorznabCatType.XXXOther, "XXX Hentai"); + caps.Categories.AddCategoryMapping(5005000, TorznabCatType.XXXOther, "XXX Other"); + caps.Categories.AddCategoryMapping(6000000, TorznabCatType.TVAnime, "Anime"); + caps.Categories.AddCategoryMapping(6001000, TorznabCatType.TVAnime, "Anime Subbed"); + caps.Categories.AddCategoryMapping(6002000, TorznabCatType.TVAnime, "Anime Dubbed"); + caps.Categories.AddCategoryMapping(6003000, TorznabCatType.TVAnime, "Anime Dual audio"); + caps.Categories.AddCategoryMapping(6004000, TorznabCatType.TVAnime, "Anime Raw"); + caps.Categories.AddCategoryMapping(6005000, TorznabCatType.AudioVideo, "Music Video"); + caps.Categories.AddCategoryMapping(6006000, TorznabCatType.BooksOther, "Literature"); + caps.Categories.AddCategoryMapping(6007000, TorznabCatType.AudioOther, "Music"); + caps.Categories.AddCategoryMapping(6008000, TorznabCatType.TVAnime, "Anime non-english translated"); + caps.Categories.AddCategoryMapping(7000000, TorznabCatType.Console, "Console"); + caps.Categories.AddCategoryMapping(7001000, TorznabCatType.ConsolePS4, "PS4"); + caps.Categories.AddCategoryMapping(7002000, TorznabCatType.ConsolePS3, "PS3"); + caps.Categories.AddCategoryMapping(7003000, TorznabCatType.ConsolePS3, "PS2"); + caps.Categories.AddCategoryMapping(7004000, TorznabCatType.ConsolePS3, "PS1"); + caps.Categories.AddCategoryMapping(7005000, TorznabCatType.ConsolePSVita, "PS Vita"); + caps.Categories.AddCategoryMapping(7006000, TorznabCatType.ConsolePSP, "PSP"); + caps.Categories.AddCategoryMapping(7007000, TorznabCatType.ConsoleXBox360, "Xbox 360"); + caps.Categories.AddCategoryMapping(7008000, TorznabCatType.ConsoleXBox, "Xbox"); + caps.Categories.AddCategoryMapping(7009000, TorznabCatType.ConsoleNDS, "Switch"); + caps.Categories.AddCategoryMapping(7010000, TorznabCatType.ConsoleNDS, "NDS"); + caps.Categories.AddCategoryMapping(7011000, TorznabCatType.ConsoleWii, "Wii"); + caps.Categories.AddCategoryMapping(7012000, TorznabCatType.ConsoleWiiU, "WiiU"); + caps.Categories.AddCategoryMapping(7013000, TorznabCatType.Console3DS, "3DS"); + caps.Categories.AddCategoryMapping(7014000, TorznabCatType.ConsoleWii, "GameCube"); + caps.Categories.AddCategoryMapping(7015000, TorznabCatType.ConsoleOther, "Other"); + caps.Categories.AddCategoryMapping(8000000, TorznabCatType.PCMobileOther, "Mobile"); + caps.Categories.AddCategoryMapping(8001000, TorznabCatType.PCMobileAndroid, "Android"); + caps.Categories.AddCategoryMapping(8002000, TorznabCatType.PCMobileiOS, "IOS"); + caps.Categories.AddCategoryMapping(8003000, TorznabCatType.PCMobileOther, "PC Other"); + caps.Categories.AddCategoryMapping(9000000, TorznabCatType.Books, "Books"); + caps.Categories.AddCategoryMapping(9001000, TorznabCatType.BooksEBook, "EBooks"); + caps.Categories.AddCategoryMapping(9002000, TorznabCatType.BooksComics, "Comics"); + caps.Categories.AddCategoryMapping(9003000, TorznabCatType.BooksMags, "Magazines"); + caps.Categories.AddCategoryMapping(9004000, TorznabCatType.BooksTechnical, "Technical"); + caps.Categories.AddCategoryMapping(9005000, TorznabCatType.BooksOther, "Books Other"); + caps.Categories.AddCategoryMapping(10000000, TorznabCatType.Other, "Other"); + caps.Categories.AddCategoryMapping(10001000, TorznabCatType.OtherMisc, "Other Misc"); + + return caps; + } + + public override IIndexerRequestGenerator GetRequestGenerator() + { + return new KnabenRequestGenerator(TorznabCaps); + } + + public override IParseIndexerResponse GetParser() + { + return new KnabenParser(TorznabCaps.Categories); + } + + public override async Task ApplyConfiguration(JToken configJson) + { + LoadValuesFromJson(configJson); + + var releases = await PerformQuery(new TorznabQuery()); + + await ConfigureIfOK(string.Empty, releases.Any(), () => throw new Exception("Could not find releases from this URL")); + + return IndexerConfigurationStatus.Completed; + } + } + + public class KnabenRequestGenerator : IIndexerRequestGenerator + { + private readonly TorznabCapabilities _capabilities; + + public KnabenRequestGenerator(TorznabCapabilities capabilities) + { + _capabilities = capabilities; + } + + public IndexerPageableRequestChain GetSearchRequests(TorznabQuery query) + { + var pageableRequests = new IndexerPageableRequestChain(); + + var postData = new Dictionary + { + { "order_by", "date" }, + { "order_direction", "desc" }, + { "from", 0 }, + { "size", 100 }, + { "hide_unsafe", true } + }; + + var searchString = query.GetQueryString().Trim(); + + if (searchString.IsNotNullOrWhiteSpace()) + { + postData.Add("search_type", "100%"); + postData.Add("search_field", "title"); + postData.Add("query", searchString); + } + + var categories = _capabilities.Categories.MapTorznabCapsToTrackers(query); + + if (categories.Any()) + { + postData.Add("categories", categories.Select(int.Parse).Distinct().ToArray()); + } + + pageableRequests.Add(GetPagedRequests(postData)); + + return pageableRequests; + } + + private IEnumerable GetPagedRequests(Dictionary postData) + { + var request = new WebRequest + { + Url = "https://api.knaben.eu/v1", + Type = RequestType.POST, + Headers = new Dictionary + { + { "Accept", "application/json" }, + { "Content-Type", "application/json" } + }, + RawBody = JsonConvert.SerializeObject(postData) + }; + + yield return new IndexerRequest(request); + } + } + + public class KnabenParser : IParseIndexerResponse + { + private readonly TorznabCapabilitiesCategories _categories; + + private static readonly Regex DateTimezoneRegex = new Regex(@"[+-]\d{2}:\d{2}$", RegexOptions.Compiled); + + public KnabenParser(TorznabCapabilitiesCategories categories) + { + _categories = categories; + } + + public IList ParseResponse(IndexerResponse indexerResponse) + { + var releases = new List(); + + var jsonResponse = JsonConvert.DeserializeObject(indexerResponse.Content); + + if (jsonResponse?.Hits == null) + { + return releases; + } + + var rows = jsonResponse.Hits.Where(r => r.Seeders > 0).ToList(); + + foreach (var row in rows) + { + // Not all entries have the TZ in the "date" field + var publishDate = row.Date.IsNotNullOrWhiteSpace() && !DateTimezoneRegex.IsMatch(row.Date) ? $"{row.Date}+01:00" : row.Date; + + var releaseInfo = new ReleaseInfo + { + Guid = new Uri(row.InfoUrl), + Title = row.Title, + Details = new Uri(row.InfoUrl), + Link = row.DownloadUrl.IsNotNullOrWhiteSpace() && Uri.TryCreate(row.DownloadUrl, UriKind.Absolute, out var downloadUrl) ? downloadUrl : null, + MagnetUri = row.MagnetUrl.IsNotNullOrWhiteSpace() && Uri.TryCreate(row.MagnetUrl, UriKind.Absolute, out var magnetUrl) ? magnetUrl : null, + Category = row.CategoryIds.SelectMany(cat => _categories.MapTrackerCatToNewznab(cat.ToString())).Distinct().ToList(), + InfoHash = row.InfoHash, + Size = row.Size, + Seeders = row.Seeders, + Peers = row.Leechers + row.Seeders, + PublishDate = DateTime.Parse(publishDate, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal), + DownloadVolumeFactor = 0, + UploadVolumeFactor = 1 + }; + + releases.Add(releaseInfo); + } + + return releases; + } + } + + internal sealed record KnabenResponse + { + public IReadOnlyCollection Hits { get; set; } = Array.Empty(); + } + + internal sealed record KnabenRelease + { + public string Title { get; set; } + + [JsonProperty("categoryId")] + public IReadOnlyCollection CategoryIds { get; set; } = Array.Empty(); + + [JsonProperty("hash")] + public string InfoHash { get; set; } + + [JsonProperty("details")] + public string InfoUrl { get; set; } + + [JsonProperty("link")] + public string DownloadUrl { get; set; } + + public string MagnetUrl { get; set; } + + [JsonProperty("bytes")] + public long Size { get; set; } + + public int Seeders { get; set; } + + [JsonProperty("peers")] + public int Leechers { get; set; } + + public string Date { get; set; } + } +} diff --git a/src/Jackett.Updater/Program.cs b/src/Jackett.Updater/Program.cs index b6ce406e1..7d0690233 100644 --- a/src/Jackett.Updater/Program.cs +++ b/src/Jackett.Updater/Program.cs @@ -487,6 +487,7 @@ namespace Jackett.Updater "Definitions/kikibt.yml", "Definitions/kinonavse100.yml", "Definitions/kisssub.yml", + "Definitions/knaben.yml", // migrated to C# API "Definitions/lapausetorrents.yml", "Definitions/lat-team.yml", // switch to *-API #8682 "Definitions/latinop2p.yml",