shpaq/Jackett-Jackett
1
0
Fork 0
mirror of https://github.com/Jackett/Jackett.git synced 2025-09-17 17:34:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

ThePirateBay: replace yml with c# and using TPB's API (#9593)

Browse Source
This commit is contained in:
Jonathan Trowbridge
2020-09-23 22:13:17 -04:00
committed by GitHub
parent 00d3a62cf1
commit 977279318d
4 changed files with 321 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
src/Jackett.Common/Converters/StringToLongConverter.cs Normal file
View File

@@ -0,0 +1,33 @@
using System;
using Newtonsoft.Json;
namespace Jackett.Common.Converters
{
/// <summary>
/// converts a string value to a long and vice-versa.
/// </summary>
public sealed class StringToLongConverter : JsonConverter
{
public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
=> writer.WriteValue(value.ToString());
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
{
if (reader.Value == null)
{
return null;
}
if (reader.Value is long)
{
return reader.Value;
}
return long.TryParse((string)reader.Value, out var foo)
? foo
: (long?) null;
}
public override bool CanConvert(Type objectType) => objectType == typeof(string);
}
}

211
src/Jackett.Common/Definitions/thepiratebay.yml
View File

@@ -1,211 +0,0 @@
---
id: thepiratebay
name: The Pirate Bay
description: "Pirate Bay (TPB) is the galaxy’s most resilient Public BitTorrent site"
language: en-us
type: public
encoding: UTF-8
followredirect: true
links:
- https://thepiratebay0.org/
- https://thepiratebay10.org/
- https://pirateproxy.live/
- https://thehiddenbay.com/
- https://thepiratebay.zone/
- https://tpb.party/
- https://piratebayproxy.live/
- https://piratebay.live/
legacylinks:
- https://thepiratesbay.pw/
- https://tproxy.pro/
- https://thepiratebay.org/ # ERR_CONNECTION_REFUSED
- https://tpb.root.yt/?ckattempt=1/ # using new html and has broken search
- https://tpb.bike/ # broken search
- https://pirateproxy.page/ # using new html
- https://tpb14.ukpass.co/ # using new html and has broken search
- https://thepiratebay.vip/ # 522
- https://piratebay1.live/ # host not found
- https://piratebay1.xyz/
- https://piratebay1.top/
- https://piratebay1.info/
- https://thepiratebay1.com/
- https://thepiratebay1.live/
- https://thepiratebays.info/
- https://thepiratebays.live/
- https://thepiratebay1.top/
- https://thepiratebay1.info/
caps:
categorymappings:
# Audio
- {id: 100, cat: Audio, desc: "Audio", default: true}
- {id: 101, cat: Audio, desc: "Music", default: true}
- {id: 102, cat: Audio/Audiobook, desc: "Audio books", default: true}
- {id: 103, cat: Audio, desc: "Sound clips", default: true}
- {id: 104, cat: Audio/Lossless, desc: "FLAC", default: true}
- {id: 199, cat: Audio/Other, desc: "Audio Other", default: true}
# Video
- {id: 200, cat: Movies, desc: "Video", default: true}
- {id: 201, cat: Movies, desc: "Movies", default: true}
- {id: 202, cat: Movies/DVD, desc: "Movies DVDR", default: true}
- {id: 203, cat: Audio/Video, desc: "Music videos", default: true}
- {id: 204, cat: Movies/Other, desc: "Movie clips", default: true}
- {id: 205, cat: TV, desc: "TV shows", default: true}
- {id: 206, cat: TV/Other, desc: "Handheld", default: true}
- {id: 207, cat: Movies/HD, desc: "HD - Movies", default: true}
- {id: 208, cat: TV/HD, desc: "HD - TV shows", default: true}
- {id: 209, cat: Movies/3D, desc: "3D", default: true}
- {id: 299, cat: Movies/Other, desc: "Video Other", default: true}
# Applications
- {id: 300, cat: PC, desc: "Applications", default: true}
- {id: 301, cat: PC, desc: "Windows", default: true}
- {id: 302, cat: PC/Mac, desc: "Mac", default: true}
- {id: 303, cat: PC, desc: "UNIX", default: true}
- {id: 304, cat: PC/Phone-Other, desc: "Handheld", default: true}
- {id: 305, cat: PC/Phone-IOS, desc: "IOS (iPad/iPhone)", default: true}
- {id: 306, cat: PC/Phone-Android, desc: "Android", default: true}
- {id: 399, cat: PC, desc: "Other OS", default: true}
# Games
- {id: 400, cat: Console, desc: "Games", default: true}
- {id: 401, cat: PC/Games, desc: "PC", default: true}
- {id: 402, cat: PC/Mac, desc: "Mac", default: true}
- {id: 403, cat: Console/PS4, desc: "PSx", default: true}
- {id: 404, cat: Console/Xbox, desc: "XBOX360", default: true}
- {id: 405, cat: Console/Wii, desc: "Wii", default: true}
- {id: 406, cat: Console/Other, desc: "Handheld", default: true}
- {id: 407, cat: Console/Other, desc: "IOS (iPad/iPhone)", default: true}
- {id: 408, cat: Console/Other, desc: "Android", default: true}
- {id: 499, cat: Console/Other, desc: "Games Other", default: true}
# Porn
- {id: 500, cat: XXX, desc: "Porn", default: true}
- {id: 501, cat: XXX, desc: "Movies", default: true}
- {id: 502, cat: XXX/DVD, desc: "Movies DVDR", default: true}
- {id: 503, cat: XXX/Imageset, desc: "Pictures", default: true}
- {id: 504, cat: XXX, desc: "Games", default: true}
- {id: 505, cat: XXX, desc: "HD - Movies", default: true}
- {id: 506, cat: XXX, desc: "Movie clips", default: true}
- {id: 599, cat: XXX/Other, desc: "Porn Other", default: true}
# Other
- {id: 600, cat: Other, desc: "Other", default: true}
- {id: 601, cat: Books, desc: "E-books", default: true}
- {id: 602, cat: Books/Comics, desc: "Comics", default: true}
- {id: 603, cat: Books, desc: "Pictures", default: true}
- {id: 604, cat: Books, desc: "Covers", default: true}
- {id: 605, cat: Books, desc: "Physibles", default: true}
- {id: 699, cat: Books/Other, desc: "Other Other", default: true}
modes:
search: [q]
tv-search: [q, season, ep]
movie-search: [q]
music-search: [q]
settings:
- name: info
type: info
label: Proxies
default: This indexer is no longer capable of processing the ThePirateBay.org domain web site due to changes the site has made to their results presentation methods.</br>But you can use any of the unofficial proxies above, which all fetch the same data that the original domain uses, only they continue to present it in a format this indexer can process.
- name: order
type: select
label: Order results
default: "99"
options:
"99": "Default"
"8": "Seeders"
"3": "Time"
search:
paths:
- path: "{{ if .Keywords }}/search/{{ .Keywords }}/0/{{ .Config.order }}/{{ join .Categories \",\"}}{{else}}/recent{{end}}"
keywordsfilters:
# remove it's #8829
- name: re_replace
args: ["(?i)(it's\\s)", ""]
# replace simplified chinese as this confuses TPB search engine #7291
- name: re_replace
args: ["([\\p{IsCJKUnifiedIdeographs}\\W]+)", "."]
# currently, the only uploader for General Hospital puts a space between season and episode
# this filter searches both formats, so "General Hospital S01E02" becomes "General Hospital S01E02 | (S01 E02)"
- name: re_replace
args: ["General Hospital S(\\d{2,3})E(\\d{2,3})", "$0 | \\(S$1 E$2\\)"]
- name: tolower
rows:
selector: "#searchResult tbody tr:has(td.vertTh)"
fields:
category: # parent category
selector: td:nth-child(1) a:first-child
attribute: href
filters:
- name: split
args: ["/", -1]
category: # sub category
selector: td:nth-child(1) a:last-child
attribute: href
filters:
- name: split
args: ["/", -1]
title:
selector: .detLink
filters:
- name: replace
args: ["\u000f", ""] # get rid of unwanted character (Example: at the end of https://thepiratebay.org/torrent/18316540/Game.of.Thrones.S07E04.iNTERNAL.1080p.WEBRip.x264-MOROSE_)
details:
selector: .detLink
attribute: href
download:
selector: td:nth-child(2) a[title^="Download"]
attribute: href
size:
selector: td:nth-child(2) font.detDesc
filters:
- name: regexp
args: "Size (.+?),"
date:
optional: true
# Today 17:37
# Y-day 13:04
selector: td:nth-child(2) font.detDesc:contains(":")
filters:
- name: regexp
args: "Uploaded (.+?),"
- name: replace
args: ["\xA0", " "]
- name: replace
args: ["Y-day", "Yesterday"]
date:
optional: true
# 49 mins ago
selector: td:nth-child(2) font.detDesc:contains("ago")
filters:
- name: regexp
args: "Uploaded (.+?),"
- name: replace
args: ["\xA0", " "]
date:
optional: true
# 04-30 2017
selector: td:nth-child(2) font.detDesc:not(:contains("ago")):not(:contains(":"))
filters:
- name: regexp
args: "Uploaded (.+?),"
- name: replace
args: ["\xA0", " "]
- name: dateparse
args: "01-02 2006"
seeders:
selector: td:nth-child(3)
leechers:
selector: td:nth-child(4)
downloadvolumefactor:
text: 0
uploadvolumefactor:
text: 1
# engine n/a

287
src/Jackett.Common/Indexers/ThePirateBay.cs Normal file
View File

@@ -0,0 +1,287 @@
using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Jackett.Common.Converters;
using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NLog;
namespace Jackett.Common.Indexers
{
/// <summary>
/// The Pirate Bay via API.
/// </summary>
[ExcludeFromCodeCoverage]
public class ThePirateBay : BaseWebIndexer
{
public override string[] AlternativeSiteLinks { get; protected set; } = {
"https://thepiratebay.org/",
"https://pirateproxy.cloud/",
"https://tpb18.ukpass.co/",
"https://tpb.sadzawka.tk/",
"https://www.tpbay.win/",
"https://tpb.cnp.cx/",
"https://thepiratebay.d4.re/",
"https://baypirated.site/",
"https://tpb.skynetcloud.site/",
"https://piratetoday.xyz/",
"https://piratenow.xyz/",
"https://piratesbaycc.com/",
};
public override string[] LegacySiteLinks { get; protected set; } = {
"https://thepiratebay0.org/",
"https://thepiratebay10.org/",
"https://pirateproxy.live/",
"https://thehiddenbay.com/",
"https://thepiratebay.zone/",
"https://tpb.party/",
"https://piratebayproxy.live/",
"https://piratebay.live/",
"https://tpb.biz/",
"https://pirate.johnedwarddoyle.co.uk/",
"https://knaben.ru/",
"https://piratebayztemzmv.onion.pet/",
"https://piratebayztemzmv.onion.ly/",
};
private const string KeyInfoHash = "{info_hash}";
private static readonly Uri _ApiBaseUri = new Uri("https://apibay.org/");
private static readonly string _MagnetUri =
$"magnet:?xt=urn:btih:{KeyInfoHash}&tr=udp%3A%2F%2Ftracker.coppersurfer.tk" +
"%3A6969%2Fannounce&tr=udp%3A%2F%2F9.rarbg.to%3A2920%2Fannounce&tr=udp%3" +
"A%2F%2Ftracker.opentrackr.org%3A1337&tr=udp%3A%2F%2Ftracker.internetwar" +
"riors.net%3A1337%2Fannounce&tr=udp%3A%2F%2Ftracker.leechers-paradise.or" +
"g%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969%2Fann" +
"ounce&tr=udp%3A%2F%2Ftracker.pirateparty.gr%3A6969%2Fannounce&tr=udp%3A" +
"%2F%2Ftracker.cyberia.is%3A6969%2Fannounce";
public ThePirateBay(
IIndexerConfigurationService configService,
WebClient client,
Logger logger,
IProtectionService p
) : base(
id: "thepiratebay",
name: "The Pirate Bay",
description: "Pirate Bay (TPB) is the galaxy’s most resilient Public BitTorrent site",
link: "https://thepiratebay.org/",
caps: new TorznabCapabilities(),
configService: configService,
client: client,
logger: logger,
p: p,
configData: new ConfigurationData()
)
{
Encoding = Encoding.UTF8;
Language = "en-us";
Type = "public";
AddCategoryMappings();
}
private void AddCategoryMappings()
{
// Audio
AddCategoryMapping(100, TorznabCatType.Audio, "Audio");
AddCategoryMapping(101, TorznabCatType.Audio, "Music");
AddCategoryMapping(102, TorznabCatType.AudioAudiobook, "Audio Books");
AddCategoryMapping(103, TorznabCatType.Audio, "Sound Clips");
AddCategoryMapping(104, TorznabCatType.AudioLossless, "FLAC");
AddCategoryMapping(199, TorznabCatType.AudioOther, "Audio Other");
// Video
AddCategoryMapping(200, TorznabCatType.Movies, "Video");
AddCategoryMapping(201, TorznabCatType.Movies, "Movies");
AddCategoryMapping(202, TorznabCatType.Movies, "Movies");
AddCategoryMapping(203, TorznabCatType.AudioVideo, "Music Videos");
AddCategoryMapping(204, TorznabCatType.MoviesOther, "Movie Clips");
AddCategoryMapping(205, TorznabCatType.TV, "TV");
AddCategoryMapping(206, TorznabCatType.TVOTHER, "Handheld");
AddCategoryMapping(207, TorznabCatType.MoviesHD, "HD - Movies");
AddCategoryMapping(208, TorznabCatType.TVHD, "HD - TV shows");
AddCategoryMapping(209, TorznabCatType.Movies3D, "3D");
AddCategoryMapping(299, TorznabCatType.MoviesOther, "Video Other");
// Applications
AddCategoryMapping(300, TorznabCatType.PC, "Applications");
AddCategoryMapping(301, TorznabCatType.PC, "Windows");
AddCategoryMapping(302, TorznabCatType.PCMac, "Mac");
AddCategoryMapping(303, TorznabCatType.PC, "UNIX");
AddCategoryMapping(304, TorznabCatType.PCPhoneOther, "Handheld");
AddCategoryMapping(305, TorznabCatType.PCPhoneIOS, "IOS (iPad/iPhone)");
AddCategoryMapping(306, TorznabCatType.PCPhoneAndroid, "Android");
AddCategoryMapping(399, TorznabCatType.PC, "Other OS");
// Games
AddCategoryMapping(400, TorznabCatType.Console, "Games");
AddCategoryMapping(401, TorznabCatType.PCGames, "PC");
AddCategoryMapping(402, TorznabCatType.PCMac, "Mac");
AddCategoryMapping(403, TorznabCatType.ConsolePS4, "PSx");
AddCategoryMapping(404, TorznabCatType.ConsoleXbox, "XBOX360");
AddCategoryMapping(405, TorznabCatType.ConsoleWii, "Wii");
AddCategoryMapping(406, TorznabCatType.ConsoleOther, "Handheld");
AddCategoryMapping(407, TorznabCatType.ConsoleOther, "IOS (iPad/iPhone)");
AddCategoryMapping(408, TorznabCatType.ConsoleOther, "Android");
AddCategoryMapping(499, TorznabCatType.ConsoleOther, "Games Other");
// Porn
AddCategoryMapping(500, TorznabCatType.XXX, "Porn");
AddCategoryMapping(501, TorznabCatType.XXX, "Movies");
AddCategoryMapping(502, TorznabCatType.XXXDVD, "Movies DVDR");
AddCategoryMapping(503, TorznabCatType.XXXImageset, "Pictures");
AddCategoryMapping(504, TorznabCatType.XXX, "Games");
AddCategoryMapping(505, TorznabCatType.XXX, "HD - Movies");
AddCategoryMapping(506, TorznabCatType.XXX, "Movie Clips");
AddCategoryMapping(599, TorznabCatType.XXXOther, "Porn other");
// Other
AddCategoryMapping(600, TorznabCatType.Other, "Other");
AddCategoryMapping(601, TorznabCatType.Books, "E-books");
AddCategoryMapping(602, TorznabCatType.BooksComics, "Comics");
AddCategoryMapping(603, TorznabCatType.Books, "Pictures");
AddCategoryMapping(604, TorznabCatType.Books, "Covers");
AddCategoryMapping(605, TorznabCatType.Books, "Physibles");
AddCategoryMapping(699, TorznabCatType.BooksOther, "Other Other");
}
public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
{
LoadValuesFromJson(configJson);
var releases = await PerformQuery(new TorznabQuery());
await ConfigureIfOK(
string.Empty,
releases.Any(),
() => throw new Exception("Could not find releases from this URL")
);
return IndexerConfigurationStatus.Completed;
}
protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
{
// Keywordless search terms return recent torrents rather than no results.
if (string.IsNullOrEmpty(query.SearchTerm))
return await GetRecentTorrents();
var categories = MapTorznabCapsToTrackers(query);
var queryStringCategories = string.Join(
",",
categories.Count == 0
? GetAllTrackerCategories()
: categories
);
var queryCollection = new NameValueCollection
{
{ "q", query.SearchTerm },
{ "cat", queryStringCategories }
};
var response = await RequestWithCookiesAsync(
$"{_ApiBaseUri}q.php?{queryCollection.GetQueryString()}"
);
var queryResponseItems = JsonConvert.DeserializeObject<List<QueryResponseItem>>(response.ContentString);
// The API returns a single item to represent a state of no results. Avoid returning this result.
if (queryResponseItems.Count == 1 && queryResponseItems.First().Id == 0)
return Enumerable.Empty<ReleaseInfo>();
return queryResponseItems.Select(CreateReleaseInfo);
}
private async Task<IEnumerable<ReleaseInfo>> GetRecentTorrents()
{
var response = await RequestWithCookiesAsync($"{_ApiBaseUri}precompiled/data_top100_recent.json");
return JsonConvert
.DeserializeObject<List<QueryResponseItem>>(response.ContentString)
.Select(CreateReleaseInfo);
}
private ReleaseInfo CreateReleaseInfo(QueryResponseItem item)
{
var magnetUri = new Uri(_MagnetUri.Replace(KeyInfoHash, item.InfoHash));
return new ReleaseInfo
{
Title = item.Name,
Category = MapTrackerCatToNewznab(item.Category.ToString()),
Comments = item.Id == 0
? null
: new Uri($"{SiteLink}description.php?id={item.Id}"),
MagnetUri = magnetUri,
InfoHash = item.InfoHash,
PublishDate = DateTimeUtil.UnixTimestampToDateTime(item.Added),
Guid = magnetUri,
Seeders = item.Seeders,
Peers = item.Seeders + item.Leechers,
Size = item.Size,
Files = item.NumFiles,
Author = item.Username,
DownloadVolumeFactor = 0,
UploadVolumeFactor = 1,
Imdb = string.IsNullOrEmpty(item.Imdb)
? null
: ParseUtil.GetImdbID(item.Imdb)
};
}
private class QueryResponseItem
{
[JsonProperty("id")]
[JsonConverter(typeof(StringToLongConverter))]
public long Id { get; set; }
[JsonProperty("name")]
public string Name { get; set; }
[JsonProperty("info_hash")]
public string InfoHash { get; set; }
[JsonProperty("leechers")]
[JsonConverter(typeof(StringToLongConverter))]
public long Leechers { get; set; }
[JsonProperty("seeders")]
[JsonConverter(typeof(StringToLongConverter))]
public long Seeders { get; set; }
[JsonProperty("num_files")]
[JsonConverter(typeof(StringToLongConverter))]
public long NumFiles { get; set; }
[JsonProperty("size")]
[JsonConverter(typeof(StringToLongConverter))]
public long Size { get; set; }
[JsonProperty("username")]
public string Username { get; set; }
[JsonProperty("added")]
[JsonConverter(typeof(StringToLongConverter))]
public long Added { get; set; }
[JsonProperty("status")]
public string Status { get; set; }
[JsonProperty("category")]
[JsonConverter(typeof(StringToLongConverter))]
public long Category { get; set; }
[JsonProperty("imdb")]
public string Imdb { get; set; }
}
}
}

1
src/Jackett.Updater/Program.cs
View File

@@ -393,6 +393,7 @@ namespace Jackett.Updater
"Definitions/theresurrection.yml",
"Definitions/thetorrents.yml",
"Definitions/the-madhouse.yml",
"Definitions/thepiratebay.yml", // migrated to c#
"Definitions/tigers-dl.yml",
"Definitions/tntvillage.yml",
"Definitions/torrentcouch.yml",