shpaq/Jackett-Jackett
1
0
Fork 0
mirror of https://github.com/Jackett/Jackett.git synced 2025-09-17 17:34:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

knaben: migrate to C# API (#15674)

Browse Source
This commit is contained in:
Bogdan
2024-10-27 19:51:15 +02:00
committed by GitHub
parent 84fbcbb82d
commit 564d390473
3 changed files with 311 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

160
src/Jackett.Common/Definitions/knaben.yml
View File

@@ -1,160 +0,0 @@
---
id: knaben
name: Knaben
description: "Knaben is a Public torrent meta-search engine"
language: en-US
type: public
encoding: UTF-8
links:
- https://knaben.eu/
caps:
categorymappings:
- {id: 1000000, cat: Audio, desc: "Audio"}
- {id: 1001000, cat: Audio/MP3, desc: "MP3"}
- {id: 1002000, cat: Audio/Lossless, desc: "Lossless"}
- {id: 1003000, cat: Audio/Audiobook, desc: "Audiobook"}
- {id: 1004000, cat: Audio/Video, desc: "Audio Video"}
- {id: 1005000, cat: Audio/Other, desc: "Radio"}
- {id: 1006000, cat: Audio/Other, desc: "Audio Other"}
- {id: 2000000, cat: TV, desc: "TV"}
- {id: 2001000, cat: TV/HD, desc: "TV HD"}
- {id: 2002000, cat: TV/SD, desc: "TV SD"}
- {id: 2003000, cat: TV/UHD, desc: "TV UHD"}
- {id: 2004000, cat: TV/Documentary, desc: "Documentary"}
- {id: 2005000, cat: TV/Foreign, desc: "TV Foreign"}
- {id: 2006000, cat: TV/Sport, desc: "Sport"}
- {id: 2007000, cat: TV/Other, desc: "Cartoon"}
- {id: 2008000, cat: TV/Other, desc: "TV Other"}
- {id: 3000000, cat: Movies, desc: "Movies"}
- {id: 3001000, cat: Movies/HD, desc: "Movies HD"}
- {id: 3002000, cat: Movies/SD, desc: "Movies SD"}
- {id: 3003000, cat: Movies/UHD, desc: "Movies UHD"}
- {id: 3004000, cat: Movies/DVD, desc: "Movies DVD"}
- {id: 3005000, cat: Movies/Foreign, desc: "Movies Foreign"}
- {id: 3006000, cat: Movies/Foreign, desc: "Movies Bollywood"}
- {id: 3007000, cat: Movies/3D, desc: "Movies 3D"}
- {id: 3008000, cat: Movies/Other, desc: "Movies Other"}
- {id: 4000000, cat: PC, desc: "PC"}
- {id: 4001000, cat: PC/Games, desc: "Games"}
- {id: 4002000, cat: PC/0day, desc: "Software"}
- {id: 4003000, cat: PC/Mac, desc: "Mac"}
- {id: 4004000, cat: PC/ISO, desc: "Unix"}
- {id: 5000000, cat: XXX, desc: "XXX"}
- {id: 5001000, cat: XXX/x264, desc: "XXX Video"}
- {id: 5002000, cat: XXX/ImageSet, desc: "XXX ImageSet"}
- {id: 5003000, cat: XXX/Other, desc: "XXX Games"}
- {id: 5004000, cat: XXX/Other, desc: "XXX Hentai"}
- {id: 5005000, cat: XXX/Other, desc: "XXX Other"}
- {id: 6000000, cat: TV/Anime, desc: "Anime"}
- {id: 6001000, cat: TV/Anime, desc: "Anime Subbed"}
- {id: 6002000, cat: TV/Anime, desc: "Anime Dubbed"}
- {id: 6003000, cat: TV/Anime, desc: "Anime Dual audio"}
- {id: 6004000, cat: TV/Anime, desc: "Anime Raw"}
- {id: 6005000, cat: Audio/Video, desc: "Music Video"}
- {id: 6006000, cat: Books/Other, desc: "Literature"}
- {id: 6007000, cat: Audio/Other, desc: "Music"}
- {id: 6008000, cat: TV/Anime, desc: "Anime non-english translated"}
- {id: 7000000, cat: Console, desc: "Console"}
- {id: 7001000, cat: Console/PS4, desc: "PS4"}
- {id: 7002000, cat: Console/PS3, desc: "PS3"}
- {id: 7003000, cat: Console/PS3, desc: "PS2"}
- {id: 7004000, cat: Console/PS3, desc: "PS1"}
- {id: 7005000, cat: Console/PS Vita, desc: "PS Vita"}
- {id: 7006000, cat: Console/PSP, desc: "PSP"}
- {id: 7007000, cat: Console/XBox 360, desc: "Xbox 360"}
- {id: 7008000, cat: Console/XBox, desc: "Xbox"}
- {id: 7009000, cat: Console/NDS, desc: "Switch"}
- {id: 7010000, cat: Console/NDS, desc: "NDS"}
- {id: 7011000, cat: Console/Wii, desc: "Wii"}
- {id: 7012000, cat: Console/WiiU, desc: "WiiU"}
- {id: 7013000, cat: Console/3DS, desc: "3DS"}
- {id: 7014000, cat: Console/Wii, desc: "GameCube"}
- {id: 7015000, cat: Console/Other, desc: "Other"}
- {id: 8000000, cat: PC/Mobile-Other, desc: "Mobile"}
- {id: 8001000, cat: PC/Mobile-Android, desc: "Android"}
- {id: 8002000, cat: PC/Mobile-iOS, desc: "IOS"}
- {id: 8003000, cat: PC/Mobile-Other, desc: "PC Other"}
- {id: 9000000, cat: Books, desc: "Books"}
- {id: 9001000, cat: Books/EBook, desc: "EBooks"}
- {id: 9002000, cat: Books/Comics, desc: "Comics"}
- {id: 9003000, cat: Books/Mags, desc: "Magazines"}
- {id: 9004000, cat: Books/Technical, desc: "Technical"}
- {id: 9005000, cat: Books/Other, desc: "Books Other"}
- {id: 10000000, cat: Other, desc: "Other"}
- {id: 10001000, cat: Other/Misc, desc: "Other Misc"}
modes:
search: [q]
tv-search: [q, season, ep]
movie-search: [q]
music-search: [q]
book-search: [q]
settings:
- name: sort
type: select
label: Sort requested from site
default: date
options:
date: created
seeders: seeders
bytes: size
title: title
- name: type
type: select
label: Order requested from site
default: "+"
options:
"+": desc
"-": asc
search:
paths:
- path: "{{ if .Keywords }}search/{{ .Keywords }}{{ else }}browse{{ end }}/0/1/{{ .Config.type }}{{ .Config.sort }}"
rows:
selector: tr[title^="Cached "], tr[title^="Checked "]
fields:
category:
selector: a[href^="/browse/"]:last-child
attribute: href
filters:
- name: split
args: ["/", 2]
# replace subcat with cat: 1001069 > 1001000
- name: re_replace
args: ["(\\d{3})$", "000"]
title:
selector: td:nth-child(2) a
details:
selector: td:last-child a
attribute: href
download:
selector: td:nth-child(2) a
attribute: href
description:
selector: td:last-child a
filters:
- name: prepend
args: "Source: "
date:
selector: td:nth-child(4)
attribute: title
filters:
- name: append
args: " +01:00" # CET
- name: dateparse
args: "yyyy-MM-dd HH:mm:ss zzz"
size:
selector: td:nth-child(3)
seeders:
selector: td:nth-child(5)
leechers:
selector: td:nth-child(6)
downloadvolumefactor:
text: 0
uploadvolumefactor:
text: 1
# engine n/a

310
src/Jackett.Common/Indexers/Definitions/Knaben.cs Normal file
View File

@@ -0,0 +1,310 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Jackett.Common.Extensions;
using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NLog;
using WebClient = Jackett.Common.Utils.Clients.WebClient;
namespace Jackett.Common.Indexers.Definitions
{
[ExcludeFromCodeCoverage]
public class Knaben : IndexerBase
{
public override string Id => "knaben";
public override string Name => "Knaben";
public override string Description => "Knaben is a Public torrent meta-search engine";
public override string SiteLink { get; protected set; } = "https://knaben.eu/";
public override string Language => "en-US";
public override string Type => "public";
public override TorznabCapabilities TorznabCaps => SetCapabilities();
public Knaben(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps, ICacheService cs)
: base(configService: configService,
client: wc,
logger: l,
p: ps,
cacheService: cs,
configData: new ConfigurationData())
{
}
private static TorznabCapabilities SetCapabilities()
{
var caps = new TorznabCapabilities
{
TvSearchParams = new List<TvSearchParam>
{
TvSearchParam.Q, TvSearchParam.Season, TvSearchParam.Ep
},
MovieSearchParams = new List<MovieSearchParam>
{
MovieSearchParam.Q
},
MusicSearchParams = new List<MusicSearchParam>
{
MusicSearchParam.Q
},
BookSearchParams = new List<BookSearchParam>
{
BookSearchParam.Q
}
};
caps.Categories.AddCategoryMapping(1000000, TorznabCatType.Audio,"Audio");
caps.Categories.AddCategoryMapping(1001000, TorznabCatType.AudioMP3, "MP3");
caps.Categories.AddCategoryMapping(1002000, TorznabCatType.AudioLossless, "Lossless");
caps.Categories.AddCategoryMapping(1003000, TorznabCatType.AudioAudiobook, "Audiobook");
caps.Categories.AddCategoryMapping(1004000, TorznabCatType.AudioVideo, "Audio Video");
caps.Categories.AddCategoryMapping(1005000, TorznabCatType.AudioOther, "Radio");
caps.Categories.AddCategoryMapping(1006000, TorznabCatType.AudioOther, "Audio Other");
caps.Categories.AddCategoryMapping(2000000, TorznabCatType.TV, "TV");
caps.Categories.AddCategoryMapping(2001000, TorznabCatType.TVHD, "TV HD");
caps.Categories.AddCategoryMapping(2002000, TorznabCatType.TVSD, "TV SD");
caps.Categories.AddCategoryMapping(2003000, TorznabCatType.TVUHD, "TV UHD");
caps.Categories.AddCategoryMapping(2004000, TorznabCatType.TVDocumentary, "Documentary");
caps.Categories.AddCategoryMapping(2005000, TorznabCatType.TVForeign, "TV Foreign");
caps.Categories.AddCategoryMapping(2006000, TorznabCatType.TVSport, "Sport");
caps.Categories.AddCategoryMapping(2007000, TorznabCatType.TVOther, "Cartoon");
caps.Categories.AddCategoryMapping(2008000, TorznabCatType.TVOther, "TV Other");
caps.Categories.AddCategoryMapping(3000000, TorznabCatType.Movies, "Movies");
caps.Categories.AddCategoryMapping(3001000, TorznabCatType.MoviesHD, "Movies HD");
caps.Categories.AddCategoryMapping(3002000, TorznabCatType.MoviesSD, "Movies SD");
caps.Categories.AddCategoryMapping(3003000, TorznabCatType.MoviesUHD, "Movies UHD");
caps.Categories.AddCategoryMapping(3004000, TorznabCatType.MoviesDVD, "Movies DVD");
caps.Categories.AddCategoryMapping(3005000, TorznabCatType.MoviesForeign, "Movies Foreign");
caps.Categories.AddCategoryMapping(3006000, TorznabCatType.MoviesForeign, "Movies Bollywood");
caps.Categories.AddCategoryMapping(3007000, TorznabCatType.Movies3D, "Movies 3D");
caps.Categories.AddCategoryMapping(3008000, TorznabCatType.MoviesOther, "Movies Other");
caps.Categories.AddCategoryMapping(4000000, TorznabCatType.PC, "PC");
caps.Categories.AddCategoryMapping(4001000, TorznabCatType.PCGames, "Games");
caps.Categories.AddCategoryMapping(4002000, TorznabCatType.PC0day, "Software");
caps.Categories.AddCategoryMapping(4003000, TorznabCatType.PCMac, "Mac");
caps.Categories.AddCategoryMapping(4004000, TorznabCatType.PCISO, "Unix");
caps.Categories.AddCategoryMapping(5000000, TorznabCatType.XXX, "XXX");
caps.Categories.AddCategoryMapping(5001000, TorznabCatType.XXXx264, "XXX Video");
caps.Categories.AddCategoryMapping(5002000, TorznabCatType.XXXImageSet, "XXX ImageSet");
caps.Categories.AddCategoryMapping(5003000, TorznabCatType.XXXOther, "XXX Games");
caps.Categories.AddCategoryMapping(5004000, TorznabCatType.XXXOther, "XXX Hentai");
caps.Categories.AddCategoryMapping(5005000, TorznabCatType.XXXOther, "XXX Other");
caps.Categories.AddCategoryMapping(6000000, TorznabCatType.TVAnime, "Anime");
caps.Categories.AddCategoryMapping(6001000, TorznabCatType.TVAnime, "Anime Subbed");
caps.Categories.AddCategoryMapping(6002000, TorznabCatType.TVAnime, "Anime Dubbed");
caps.Categories.AddCategoryMapping(6003000, TorznabCatType.TVAnime, "Anime Dual audio");
caps.Categories.AddCategoryMapping(6004000, TorznabCatType.TVAnime, "Anime Raw");
caps.Categories.AddCategoryMapping(6005000, TorznabCatType.AudioVideo, "Music Video");
caps.Categories.AddCategoryMapping(6006000, TorznabCatType.BooksOther, "Literature");
caps.Categories.AddCategoryMapping(6007000, TorznabCatType.AudioOther, "Music");
caps.Categories.AddCategoryMapping(6008000, TorznabCatType.TVAnime, "Anime non-english translated");
caps.Categories.AddCategoryMapping(7000000, TorznabCatType.Console, "Console");
caps.Categories.AddCategoryMapping(7001000, TorznabCatType.ConsolePS4, "PS4");
caps.Categories.AddCategoryMapping(7002000, TorznabCatType.ConsolePS3, "PS3");
caps.Categories.AddCategoryMapping(7003000, TorznabCatType.ConsolePS3, "PS2");
caps.Categories.AddCategoryMapping(7004000, TorznabCatType.ConsolePS3, "PS1");
caps.Categories.AddCategoryMapping(7005000, TorznabCatType.ConsolePSVita, "PS Vita");
caps.Categories.AddCategoryMapping(7006000, TorznabCatType.ConsolePSP, "PSP");
caps.Categories.AddCategoryMapping(7007000, TorznabCatType.ConsoleXBox360, "Xbox 360");
caps.Categories.AddCategoryMapping(7008000, TorznabCatType.ConsoleXBox, "Xbox");
caps.Categories.AddCategoryMapping(7009000, TorznabCatType.ConsoleNDS, "Switch");
caps.Categories.AddCategoryMapping(7010000, TorznabCatType.ConsoleNDS, "NDS");
caps.Categories.AddCategoryMapping(7011000, TorznabCatType.ConsoleWii, "Wii");
caps.Categories.AddCategoryMapping(7012000, TorznabCatType.ConsoleWiiU, "WiiU");
caps.Categories.AddCategoryMapping(7013000, TorznabCatType.Console3DS, "3DS");
caps.Categories.AddCategoryMapping(7014000, TorznabCatType.ConsoleWii, "GameCube");
caps.Categories.AddCategoryMapping(7015000, TorznabCatType.ConsoleOther, "Other");
caps.Categories.AddCategoryMapping(8000000, TorznabCatType.PCMobileOther, "Mobile");
caps.Categories.AddCategoryMapping(8001000, TorznabCatType.PCMobileAndroid, "Android");
caps.Categories.AddCategoryMapping(8002000, TorznabCatType.PCMobileiOS, "IOS");
caps.Categories.AddCategoryMapping(8003000, TorznabCatType.PCMobileOther, "PC Other");
caps.Categories.AddCategoryMapping(9000000, TorznabCatType.Books, "Books");
caps.Categories.AddCategoryMapping(9001000, TorznabCatType.BooksEBook, "EBooks");
caps.Categories.AddCategoryMapping(9002000, TorznabCatType.BooksComics, "Comics");
caps.Categories.AddCategoryMapping(9003000, TorznabCatType.BooksMags, "Magazines");
caps.Categories.AddCategoryMapping(9004000, TorznabCatType.BooksTechnical, "Technical");
caps.Categories.AddCategoryMapping(9005000, TorznabCatType.BooksOther, "Books Other");
caps.Categories.AddCategoryMapping(10000000, TorznabCatType.Other, "Other");
caps.Categories.AddCategoryMapping(10001000, TorznabCatType.OtherMisc, "Other Misc");
return caps;
}
public override IIndexerRequestGenerator GetRequestGenerator()
{
return new KnabenRequestGenerator(TorznabCaps);
}
public override IParseIndexerResponse GetParser()
{
return new KnabenParser(TorznabCaps.Categories);
}
public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
{
LoadValuesFromJson(configJson);
var releases = await PerformQuery(new TorznabQuery());
await ConfigureIfOK(string.Empty, releases.Any(), () => throw new Exception("Could not find releases from this URL"));
return IndexerConfigurationStatus.Completed;
}
}
public class KnabenRequestGenerator : IIndexerRequestGenerator
{
private readonly TorznabCapabilities _capabilities;
public KnabenRequestGenerator(TorznabCapabilities capabilities)
{
_capabilities = capabilities;
}
public IndexerPageableRequestChain GetSearchRequests(TorznabQuery query)
{
var pageableRequests = new IndexerPageableRequestChain();
var postData = new Dictionary<string, object>
{
{ "order_by", "date" },
{ "order_direction", "desc" },
{ "from", 0 },
{ "size", 100 },
{ "hide_unsafe", true }
};
var searchString = query.GetQueryString().Trim();
if (searchString.IsNotNullOrWhiteSpace())
{
postData.Add("search_type", "100%");
postData.Add("search_field", "title");
postData.Add("query", searchString);
}
var categories = _capabilities.Categories.MapTorznabCapsToTrackers(query);
if (categories.Any())
{
postData.Add("categories", categories.Select(int.Parse).Distinct().ToArray());
}
pageableRequests.Add(GetPagedRequests(postData));
return pageableRequests;
}
private IEnumerable<IndexerRequest> GetPagedRequests(Dictionary<string, object> postData)
{
var request = new WebRequest
{
Url = "https://api.knaben.eu/v1",
Type = RequestType.POST,
Headers = new Dictionary<string, string>
{
{ "Accept", "application/json" },
{ "Content-Type", "application/json" }
},
RawBody = JsonConvert.SerializeObject(postData)
};
yield return new IndexerRequest(request);
}
}
public class KnabenParser : IParseIndexerResponse
{
private readonly TorznabCapabilitiesCategories _categories;
private static readonly Regex DateTimezoneRegex = new Regex(@"[+-]\d{2}:\d{2}$", RegexOptions.Compiled);
public KnabenParser(TorznabCapabilitiesCategories categories)
{
_categories = categories;
}
public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
{
var releases = new List<ReleaseInfo>();
var jsonResponse = JsonConvert.DeserializeObject<KnabenResponse>(indexerResponse.Content);
if (jsonResponse?.Hits == null)
{
return releases;
}
var rows = jsonResponse.Hits.Where(r => r.Seeders > 0).ToList();
foreach (var row in rows)
{
// Not all entries have the TZ in the "date" field
var publishDate = row.Date.IsNotNullOrWhiteSpace() && !DateTimezoneRegex.IsMatch(row.Date) ? $"{row.Date}+01:00" : row.Date;
var releaseInfo = new ReleaseInfo
{
Guid = new Uri(row.InfoUrl),
Title = row.Title,
Details = new Uri(row.InfoUrl),
Link = row.DownloadUrl.IsNotNullOrWhiteSpace() && Uri.TryCreate(row.DownloadUrl, UriKind.Absolute, out var downloadUrl) ? downloadUrl : null,
MagnetUri = row.MagnetUrl.IsNotNullOrWhiteSpace() && Uri.TryCreate(row.MagnetUrl, UriKind.Absolute, out var magnetUrl) ? magnetUrl : null,
Category = row.CategoryIds.SelectMany(cat => _categories.MapTrackerCatToNewznab(cat.ToString())).Distinct().ToList(),
InfoHash = row.InfoHash,
Size = row.Size,
Seeders = row.Seeders,
Peers = row.Leechers + row.Seeders,
PublishDate = DateTime.Parse(publishDate, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal),
DownloadVolumeFactor = 0,
UploadVolumeFactor = 1
};
releases.Add(releaseInfo);
}
return releases;
}
}
internal sealed record KnabenResponse
{
public IReadOnlyCollection<KnabenRelease> Hits { get; set; } = Array.Empty<KnabenRelease>();
}
internal sealed record KnabenRelease
{
public string Title { get; set; }
[JsonProperty("categoryId")]
public IReadOnlyCollection<int> CategoryIds { get; set; } = Array.Empty<int>();
[JsonProperty("hash")]
public string InfoHash { get; set; }
[JsonProperty("details")]
public string InfoUrl { get; set; }
[JsonProperty("link")]
public string DownloadUrl { get; set; }
public string MagnetUrl { get; set; }
[JsonProperty("bytes")]
public long Size { get; set; }
public int Seeders { get; set; }
[JsonProperty("peers")]
public int Leechers { get; set; }
public string Date { get; set; }
}
}

1
src/Jackett.Updater/Program.cs
View File

@@ -487,6 +487,7 @@ namespace Jackett.Updater
"Definitions/kikibt.yml",
"Definitions/kinonavse100.yml",
"Definitions/kisssub.yml",
"Definitions/knaben.yml", // migrated to C# API
"Definitions/lapausetorrents.yml",
"Definitions/lat-team.yml", // switch to *-API #8682
"Definitions/latinop2p.yml",