From 5f4206608e19a4a4846eb13848e9f42224286a92 Mon Sep 17 00:00:00 2001 From: fnk93 Date: Tue, 10 Mar 2020 08:11:35 +0100 Subject: [PATCH] TorrentSeeds: Convert to csharp (#7603) --- .../Definitions/torrentseeds.yml | 166 ------------ src/Jackett.Common/Indexers/Torrentseeds.cs | 256 ++++++++++++++++++ src/Jackett.Updater/Program.cs | 1 + 3 files changed, 257 insertions(+), 166 deletions(-) delete mode 100644 src/Jackett.Common/Definitions/torrentseeds.yml create mode 100644 src/Jackett.Common/Indexers/Torrentseeds.cs diff --git a/src/Jackett.Common/Definitions/torrentseeds.yml b/src/Jackett.Common/Definitions/torrentseeds.yml deleted file mode 100644 index 0bf3b91b7..000000000 --- a/src/Jackett.Common/Definitions/torrentseeds.yml +++ /dev/null @@ -1,166 +0,0 @@ ---- - site: torrentseeds - name: TorrentSeeds - description: "TorrentSeeds is a Private site for MOVIES / TV / GENERAL" - language: en-us - type: private - encoding: UTF-8 - links: - - https://www.torrentseeds.org/ - - caps: - categorymappings: - - {id: 13, cat: PC/0day, desc: "Apps/0DAY"} - - {id: 37, cat: TV/Anime, desc: "Anime/HD"} - - {id: 9, cat: TV/Anime, desc: "Anime/SD"} - - {id: 1, cat: PC/0day, desc: "Apps"} - - {id: 27, cat: Books, desc: "Bookware"} - - {id: 32, cat: Books/Ebook, desc: "EBooks"} - - {id: 47, cat: Console/Other, desc: "Games/NSW"} - - {id: 60, cat: Console/Other, desc: "Games/ATARI"} - - {id: 63, cat: Console/Other, desc: "Games/UPDATES"} - - {id: 2, cat: PC/Games, desc: "Games/PC"} - - {id: 8, cat: Console/PS3, desc: "Games/PS3"} - - {id: 30, cat: Console/PS4, desc: "Games/PS4"} - - {id: 7, cat: Console/PSP, desc: "Games/PSP"} - - {id: 16, cat: Console/Wii, desc: "Games/WII"} - - {id: 29, cat: Console/WiiU, desc: "Games/WIIU"} - - {id: 17, cat: Console/XBox 360, desc: "Games/XBOX360"} - - {id: 50, cat: Movies/BluRay, desc: "Movies/Bluray-UHD"} - - {id: 31, cat: Movies/BluRay, desc: "Movies/COMPLETE-BLURAY"} - - {id: 3, cat: Movies/DVD, desc: "Movies/DVDR"} - - {id: 39, cat: Movies/Foreign, desc: "Movies/HD-Foreign"} - - {id: 62, cat: Movies/Foreign, desc: "Movies/SD-Foreign"} - - {id: 19, cat: Movies/HD, desc: "Movies/X264"} - - {id: 49, cat: Movies/HD, desc: "Movies/X265"} - - {id: 25, cat: Movies/SD, desc: "Movies/XVID"} - - {id: 6, cat: XXX, desc: "Movies/XXX"} - - {id: 53, cat: XXX, desc: "Movies/XXX-HD"} - - {id: 57, cat: XXX, desc: "Movies/XXX-PAYSITE"} - - {id: 55, cat: XXX, desc: "Movies/XXX-DVDR"} - - {id: 33, cat: Audio/Lossless, desc: "Music/FLAC"} - - {id: 28, cat: Audio/Other, desc: "Music/MBluRay"} - - {id: 34, cat: Audio/Other, desc: "Music/MDVDR"} - - {id: 4, cat: Audio/MP3, desc: "Music/MP3"} - - {id: 20, cat: Audio/Video, desc: "Music/MVID"} - - {id: 38, cat: TV/Anime, desc: "P2P/ANIME"} - - {id: 48, cat: PC/0day, desc: "P2P/APPS"} - - {id: 43, cat: Movies/BluRay, desc: "P2P/BLURAY"} - - {id: 52, cat: Movies/BluRay, desc: "P2P/Bluray-UHD"} - - {id: 40, cat: Movies/DVD, desc: "P2P/DVDR"} - - {id: 46, cat: Books/EBook, desc: "P2P/EBOOKS"} - - {id: 45, cat: PC/Games, desc: "P2P/GAMES"} - - {id: 42, cat: Movies/HD, desc: "P2P/HD-MOVIES"} - - {id: 44, cat: TV/HD, desc: "P2P/TV-HD"} - - {id: 51, cat: Movies/HD, desc: "P2P/X265"} - - {id: 41, cat: Movies/SD, desc: "P2P/XVID"} - - {id: 35, cat: TV/Sport, desc: "TV/SPORT"} - - {id: 36, cat: TV/Sport, desc: "TV/SPORT-HD"} - - {id: 11, cat: TV/HD, desc: "TV/BluRay"} - - {id: 23, cat: TV/SD, desc: "TV/DVDR"} - - {id: 24, cat: TV/SD, desc: "TV/DVDRIP"} - - {id: 18, cat: TV/SD, desc: "TV/SD"} - - {id: 26, cat: TV/HD, desc: "TV/X264"} - - {id: 61, cat: TV/UHD, desc: "TV/2160P"} - - {id: 64, cat: TV/FOREIGN, desc: "TV/X264-FOREIGN"} - - - modes: - search: [q] - tv-search: [q, season, ep] - movie-search: [q] - - settings: - - name: username - type: text - label: Username - - name: password - type: password - label: Password - - name: info - type: info - label: Results Per Page - default: For best results, change the Torrents per page setting to 100 on your Torrent tab from the Pers Tools > UserCP menu on the TorrentSeeds webpage. - - name: sort - type: select - label: Sort requested from site - default: "added" - options: - "added": "created" - "seeders": "seeders" - "size": "size" - "name": "title" - - name: type - type: select - label: Order requested from site - default: "desc" - options: - "desc": "desc" - "asc": "asc" - - login: - path: takelogin.php - method: post - inputs: - username: "{{ .Config.username }}" - password: "{{ .Config.password }}" - remember: on - error: - - selector: h2:contains("Login failed!") - message: - selector: td.colhead2 - test: - path: browse.php - - search: - paths: - - path: browse_elastic.php - inputs: - $raw: "{{ range .Categories }}cat[{{.}}]=1&{{end}}" - query: "{{ .Keywords }}" - # name, descr, genre, all - search_in: name - # all, any - search_mode: all - order_by: "{{ .Config.sort }}" - order_way: "{{ .Config.type }}" - - rows: - selector: table.torrent-table tr[class] - - fields: - category: - selector: a[href^="/browse_elastic.php?cat="] - attribute: href - filters: - - name: querystring - args: cat - title: - selector: a[href^="/details.php?id="] - details: - selector: a[href^="/details.php?id="] - attribute: href - download: - selector: a[href^="/download.php?torrent="] - attribute: href - files: - selector: td.torrent-table-files - date: - selector: td.torrent-table-added - filters: - - name: timeago - size: - selector: td.torrent-table-size - grabs: - selector: td.torrent-table-snatched - seeders: - selector: td.torrent-table-seeders - leechers: - selector: td.torrent-table-leechers - downloadvolumefactor: - case: - span.freeleech: 0 - "*": 1 - uploadvolumefactor: - text: 1 -# U-232 V5 diff --git a/src/Jackett.Common/Indexers/Torrentseeds.cs b/src/Jackett.Common/Indexers/Torrentseeds.cs new file mode 100644 index 000000000..77a115837 --- /dev/null +++ b/src/Jackett.Common/Indexers/Torrentseeds.cs @@ -0,0 +1,256 @@ +using System; +using System.Collections.Generic; +using System.Collections.Specialized; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using AngleSharp.Html.Parser; +using Jackett.Common.Models; +using Jackett.Common.Models.IndexerConfig; +using Jackett.Common.Services.Interfaces; +using Jackett.Common.Utils; +using Jackett.Common.Utils.Clients; +using Newtonsoft.Json.Linq; +using NLog; + +namespace Jackett.Common.Indexers +{ + public class TorrentSeeds : BaseWebIndexer + { + private string LoginUrl => SiteLink + "takelogin.php"; + private string SearchUrl => SiteLink + "browse_elastic.php"; + private string TokenUrl => SiteLink + "login.php"; + + private new ConfigurationDataBasicLoginWithRSSAndDisplay configData + { + get => (ConfigurationDataBasicLoginWithRSSAndDisplay)base.configData; + set => base.configData = value; + } + + public TorrentSeeds(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps) : + base( + name: "TorrentSeeds", + description: "TorrentSeeds is a Private site for MOVIES / TV / GENERAL", + link: "https://torrentseeds.org/", + caps: TorznabUtil.CreateDefaultTorznabTVCaps(), + configService: configService, + client: wc, + logger: l, + p: ps, + configData: new ConfigurationDataBasicLoginWithRSSAndDisplay()) + { + Encoding = Encoding.UTF8; + Language = "en-us"; + Type = "private"; + AddCategoryMapping(23, TorznabCatType.PC0day, "Apps/0DAY"); + AddCategoryMapping(37, TorznabCatType.TVAnime, "Anime/HD"); + AddCategoryMapping(9, TorznabCatType.TVAnime, "Anime/SD"); + AddCategoryMapping(1, TorznabCatType.PC0day, "Apps"); + AddCategoryMapping(27, TorznabCatType.Books, "Bookware"); + AddCategoryMapping(32, TorznabCatType.BooksEbook, "EBooks"); + AddCategoryMapping(47, TorznabCatType.ConsoleOther, "Games/NSW"); + AddCategoryMapping(60, TorznabCatType.ConsoleOther, "Games/ATARI"); + AddCategoryMapping(63, TorznabCatType.ConsoleOther, "Games/UPDATES"); + AddCategoryMapping(2, TorznabCatType.PCGames, "Games/PC"); + AddCategoryMapping(8, TorznabCatType.ConsolePS3, "Games/PS3"); + AddCategoryMapping(30, TorznabCatType.ConsolePS4, "Games/PS4"); + AddCategoryMapping(7, TorznabCatType.ConsolePSP, "Games/PSP"); + AddCategoryMapping(16, TorznabCatType.ConsoleWii, "Games/WII"); + AddCategoryMapping(29, TorznabCatType.ConsoleWiiU, "Games/WIIU"); + AddCategoryMapping(17, TorznabCatType.ConsoleXbox360, "Games/XBOX360"); + AddCategoryMapping(50, TorznabCatType.MoviesBluRay, "Movies/Bluray-UHD"); + AddCategoryMapping(31, TorznabCatType.MoviesBluRay, "Movies/COMPLETE-BLURAY"); + AddCategoryMapping(3, TorznabCatType.MoviesDVD, "Movies/DVDR"); + AddCategoryMapping(39, TorznabCatType.MoviesForeign, "Movies/HD-Foreign"); + AddCategoryMapping(62, TorznabCatType.MoviesForeign, "Movies/SD-Foreign"); + AddCategoryMapping(19, TorznabCatType.MoviesHD, "Movies/X264"); + AddCategoryMapping(49, TorznabCatType.MoviesHD, "Movies/X265"); + AddCategoryMapping(25, TorznabCatType.MoviesSD, "Movies/XVID"); + AddCategoryMapping(6, TorznabCatType.XXX, "Movies/XXX"); + AddCategoryMapping(53, TorznabCatType.XXX, "Movies/XXX-HD"); + AddCategoryMapping(57, TorznabCatType.XXX, "Movies/XXX-PAYSITE"); + AddCategoryMapping(55, TorznabCatType.XXX, "Movies/XXX-DVDR"); + AddCategoryMapping(33, TorznabCatType.AudioLossless, "Music/FLAC"); + AddCategoryMapping(28, TorznabCatType.AudioOther, "Music/MBluRay"); + AddCategoryMapping(34, TorznabCatType.AudioOther, "Music/MDVDR"); + AddCategoryMapping(4, TorznabCatType.AudioMP3, "Music/MP3"); + AddCategoryMapping(20, TorznabCatType.AudioVideo, "Music/MVID"); + AddCategoryMapping(38, TorznabCatType.TVAnime, "P2P/ANIME"); + AddCategoryMapping(48, TorznabCatType.PC0day, "P2P/APPS"); + AddCategoryMapping(43, TorznabCatType.MoviesBluRay, "P2P/BLURAY"); + AddCategoryMapping(52, TorznabCatType.MoviesBluRay, "P2P/Bluray-UHD"); + AddCategoryMapping(40, TorznabCatType.MoviesDVD, "P2P/DVDR"); + AddCategoryMapping(46, TorznabCatType.BooksEbook, "P2P/EBOOKS"); + AddCategoryMapping(45, TorznabCatType.PCGames, "P2P/GAMES"); + AddCategoryMapping(42, TorznabCatType.MoviesHD, "P2P/HD-MOVIES"); + AddCategoryMapping(44, TorznabCatType.TVHD, "P2P/TV-HD"); + AddCategoryMapping(51, TorznabCatType.MoviesHD, "P2P/X265"); + AddCategoryMapping(41, TorznabCatType.MoviesSD, "P2P/XVID"); + AddCategoryMapping(35, TorznabCatType.TVSport, "TV/SPORT"); + AddCategoryMapping(36, TorznabCatType.TVSport, "TV/SPORT-HD"); + AddCategoryMapping(11, TorznabCatType.TVHD, "TV/BluRay"); + AddCategoryMapping(23, TorznabCatType.TVSD, "TV/DVDR"); + AddCategoryMapping(24, TorznabCatType.TVSD, "TV/DVDRIP"); + AddCategoryMapping(18, TorznabCatType.TVSD, "TV/SD"); + AddCategoryMapping(26, TorznabCatType.TVHD, "TV/X264"); + AddCategoryMapping(61, TorznabCatType.TVUHD, "TV/2160P"); + AddCategoryMapping(64, TorznabCatType.TVFOREIGN, "TV/X264-FOREIGN"); + } + + public override async Task ApplyConfiguration(JToken configJson) + { + LoadValuesFromJson(configJson); + var loginPage = await RequestStringWithCookies(TokenUrl); + var parser = new HtmlParser(); + var dom = parser.ParseDocument(loginPage.Content); + var token = dom.QuerySelector("form.form-horizontal > span"); + var csrf = token.Children[1].GetAttribute("value"); + var pairs = new Dictionary + { + { "username", configData.Username.Value }, + { "password", configData.Password.Value }, + { "perm_ssl", "1" }, + { "returnto", "/" }, + { "csrf_token", csrf } + }; + var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, accumulateCookies: true); + await ConfigureIfOK( + result.Cookies, result.Content?.Contains("https://torrentseeds.org/logout.php") == true && !result.Content.Contains("Login failed!"), + () => + { + var errorDom = parser.ParseDocument(result.Content); + var errorMessage = errorDom.QuerySelector("td.colhead2").InnerHtml; + throw new ExceptionWithConfigData(errorMessage, configData); + }); + return IndexerConfigurationStatus.RequiresTesting; + } + + protected override async Task> PerformQuery(TorznabQuery query) + { + var releases = new List(); + var searchString = query.GetQueryString(); + var finalSearchString = Regex.Replace(searchString.Trim(), "[ _.-]+", " ", RegexOptions.Compiled); + var searchUrl = SearchUrl; + var queryCollection = new NameValueCollection + { + { "search_in", "name" }, + { "search_mode", "all" }, + { "order_by", "added" }, + { "order_way", "desc" } + }; + if (!string.IsNullOrWhiteSpace(finalSearchString)) + queryCollection.Add("query", finalSearchString); + foreach (var cat in MapTorznabCapsToTrackers(query)) + queryCollection.Add("cat[" + cat + "]", "1"); + searchUrl += "?" + queryCollection.GetQueryString(); + var response = await RequestStringWithCookiesAndRetry(searchUrl); + var results = response.Content; + if (results.Contains("takelogin.php") || response.Cookies?.Contains("pass=deleted;") == true) + { + await ApplyConfiguration(null); + response = await RequestStringWithCookiesAndRetry(searchUrl); + } + + if (response.IsRedirect) + { + // handle single entries + var qCommentLink = new Uri(response.RedirectingTo); + await FollowIfRedirect(response, accumulateCookies: true); + results = response.Content; + try + { + var parser = new HtmlParser(); + var dom = parser.ParseDocument(results); + var content = dom.QuerySelector("tbody:has(script)"); + var release = new ReleaseInfo(); + release.MinimumRatio = 1; + release.MinimumSeedTime = 72 * 60 * 60; + var catStr = content.QuerySelector("tr:has(td:contains(\"Type\"))").Children[1].TextContent; + release.Category = MapTrackerCatDescToNewznab(catStr); + var qLink = content.QuerySelector("tr:has(td:contains(\"Download\"))") + .QuerySelector("a[href*=\"download.php?torrent=\"]"); + release.Link = new Uri(SiteLink + qLink.GetAttribute("href")); + release.Title = qLink.QuerySelector("u").TextContent; + release.Comments = qCommentLink; + release.Guid = release.Comments; + var qSize = content.QuerySelector("tr:has(td.heading:contains(\"Size\"))").Children[1].TextContent + .Split('(')[0].Trim(); + release.Size = ReleaseInfo.GetBytes(qSize); + var peerStats = content.QuerySelector("tr:has(td:has(a[href^=\"./peerlist_xbt.php?id=\"]))").Children[1] + .TextContent.Split(','); + var qSeeders = peerStats[0].Replace(" seeder(s)", "").Trim(); + var qLeechers = peerStats[1].Split('=')[0].Replace(" leecher(s) ", "").Trim(); + release.Seeders = ParseUtil.CoerceInt(qSeeders); + release.Peers = ParseUtil.CoerceInt(qLeechers) + release.Seeders; + var rawDateStr = content.QuerySelector("tr:has(td:contains(\"Added\"))").Children[1].TextContent; + var dateUpped = DateTimeUtil.FromUnknown(rawDateStr.Replace(",", string.Empty)); + + // Mar 4 2020, 05:47 AM + release.PublishDate = dateUpped.ToLocalTime(); + var qGrabs = content.QuerySelector("tr:has(td:contains(\"Snatched\"))").Children[1]; + release.Grabs = ParseUtil.CoerceInt(qGrabs.TextContent.Replace(" time(s)", "")); + var qFiles = content.QuerySelector("tr:has(td:has(a[href^=\"./filelist.php?id=\"]))").Children[1]; + release.Files = ParseUtil.CoerceInt(qFiles.TextContent.Replace(" files", "")); + var qRatio = content.QuerySelector("tr:has(td:contains(\"Ratio After Download\"))").Children[1]; + release.DownloadVolumeFactor = qRatio.QuerySelector("del") != null ? 0 : 1; + release.UploadVolumeFactor = 1; + releases.Add(release); + } + catch (Exception ex) + { + OnParseError(results, ex); + } + + return releases; + } + + try + { + var parser = new HtmlParser(); + var dom = parser.ParseDocument(results); + var rows = dom.QuerySelectorAll("table.torrent-table > tbody > tr"); + foreach (var row in rows.Skip(1)) + { + var release = new ReleaseInfo(); + release.MinimumRatio = 1; + release.MinimumSeedTime = 72 * 60 * 60; + var qCatLink = row.QuerySelector("a[href^=\"/browse_elastic.php?cat=\"]"); + var catStr = qCatLink.GetAttribute("href").Split('=')[1]; + release.Category = MapTrackerCatToNewznab(catStr); + var qDetailsLink = row.QuerySelector("a[href^=\"/details.php?id=\"]"); + var qDetailsTitle = row.QuerySelector("td:has(a[href^=\"/details.php?id=\"]) b"); + release.Title = qDetailsTitle.TextContent; + var qDlLink = row.QuerySelector("a[href^=\"/download.php?torrent=\"]"); + var qSeeders = row.QuerySelector("td.torrent-table-seeders"); + var qLeechers = row.QuerySelector("td.torrent-table-leechers"); + var qDateStr = row.QuerySelector("td.torrent-table-added"); + var qSize = row.QuerySelector("td.torrent-table-size"); + var qGrabs = row.QuerySelector("td.torrent-table-snatched"); + var qFiles = row.QuerySelector("td.torrent-table-files"); + release.Link = new Uri(SiteLink + qDlLink.GetAttribute("href").Split('/')[1]); + release.Comments = new Uri(SiteLink + qDetailsLink.GetAttribute("href").Split('/')[1]); + release.Guid = release.Comments; + var sizeStr = qSize.TextContent; + release.Size = ReleaseInfo.GetBytes(sizeStr); + release.Seeders = ParseUtil.CoerceInt(qSeeders.TextContent); + release.Peers = ParseUtil.CoerceInt(qLeechers.TextContent) + release.Seeders; + var dateStr = ParseUtil.CoerceLong(qDateStr.GetAttribute("data-timestamp")); + var parsedDateTime = DateTimeUtil.UnixTimestampToDateTime(dateStr); + release.PublishDate = parsedDateTime; + release.Grabs = ParseUtil.CoerceInt(qGrabs.TextContent); + release.Files = ParseUtil.CoerceInt(qFiles.TextContent); + release.DownloadVolumeFactor = row.GetAttribute("class").Contains("freeleech") ? 0 : 1; + release.UploadVolumeFactor = 1; + releases.Add(release); + } + } + catch (Exception ex) + { + OnParseError(results, ex); + } + + return releases; + } + } +} diff --git a/src/Jackett.Updater/Program.cs b/src/Jackett.Updater/Program.cs index c2a542745..a68c298a8 100644 --- a/src/Jackett.Updater/Program.cs +++ b/src/Jackett.Updater/Program.cs @@ -369,6 +369,7 @@ namespace Jackett.Updater "Definitions/torrentcouch.yml", "Definitions/torrentkim.yml", "Definitions/torrentproject.yml", + "Definitions/torrentseeds.yml", // migrated to c# "Definitions/torrentsmd.yml", "Definitions/torrentvault.yml", "Definitions/torrentwtf.yml",