newpct: fix download links. resolves #12914 (#12921)

README.md

@@ -61,7 +61,6 @@ A third-party Golang SDK for Jackett is available from [webtor-io/go-jackett](ht
  * elitetorrent
  * EpubLibre
  * Erai-Raws
- * ETTV
  * EXT Torrents
  * ExtraTorrent.st
  * EZTV
@@ -111,7 +110,6 @@ A third-party Golang SDK for Jackett is available from [webtor-io/go-jackett](ht
  * ParnuXi
  * PC-torrent
  * PiratBit
- * Pirateiro
  * Pornforall
  * PornLeech
  * PornoLive

src/Jackett.Common/Definitions/ettv.yml

@@ -1,181 +0,0 @@
----
-id: ettv
-name: ETTV
-description: "ETTV is a Public torrent site for TV / MOVIES, home of the  ETTV, ETHD and DTOne groups."
-language: en-US
-type: public
-encoding: UTF-8
-followredirect: true
-links:
-  - https://www.ettvcentral.com/
-  - https://ettv.unblockninja.com/
-  - https://ettv.unblockit.how/
-  - https://ettv.nocensor.biz/
-legacylinks:
-  - https://www.ettv.tv/
-  - https://www.ettv.to/
-  - https://www.ettvdl.com/
-  - https://www.ettv.be/
-  - https://ettv.unblockit.dev/
-  - https://ettv.unblockit.ltd/
-  - https://ettv.unblockit.link/
-  - https://ettv.unblockit.buzz/
-  - https://ettv.unblocked.monster/
-  - https://ettv.unblockit.club/
-  - https://ettv.unblockit.onl/
-  - https://ettv.unblockit.li/
-  - https://ettv.unblockit.uno/
-  - https://ettv.unblockit.ch/
-  - https://ettv.nocensor.space/
-  - https://ettv.unblockit.ws/
-  - https://ettv.nocensor.work/
-  - https://ettv.unblockit.kim/
-  - https://ettv.unblockit.bz/
-  - https://ettv.unblockit.tv/
-
-caps:
-  categorymappings:
-    - {id: 88, cat: XXX, desc: "Adult - Books"}
-    - {id: 84, cat: XXX, desc: "Adult - Games"}
-    - {id: 82, cat: XXX, desc: "Adult - HD-Movies"}
-    - {id: 83, cat: XXX, desc: "Adult - Hentai"}
-    - {id: 86, cat: XXX, desc: "Adult - Magazines"}
-    - {id: 81, cat: XXX, desc: "Adult - Movies"}
-    - {id: 87, cat: XXX, desc: "Adult - Other"}
-    - {id: 85, cat: XXX, desc: "Adult - Pictures"}
-    - {id: 74, cat: TV/Anime, desc: "Anime - Dubbed/Subbed"}
-    - {id: 73, cat: TV/Anime, desc: "Anime - Movies"}
-    - {id: 75, cat: TV/Anime, desc: "Anime - Others"}
-    - {id: 56, cat: Audio/Audiobook, desc: "Books - Audio"}
-    - {id: 55, cat: Books/Comics, desc: "Books - Comics"}
-    - {id: 53, cat: Books/EBook, desc: "Books - Ebooks"}
-    - {id: 54, cat: Books/Mags, desc: "Books - Magazines"}
-    - {id: 68, cat: Books/Other, desc: "Books - Others"}
-    - {id: 90, cat: Books/Other, desc: "Books - Others"}
-    - {id: 58, cat: PC/Games, desc: "Games - Android"}
-    - {id: 71, cat: PC/Games, desc: "Games - Others"}
-    - {id: 57, cat: PC/Games, desc: "Games - Windows"}
-    - {id: 49, cat: Movies/3D, desc: "Movies - 3D"}
-    - {id: 66, cat: Movies/BluRay, desc: "Movies - BluRay Disc/Remux"}
-    - {id: 91, cat: Movies, desc: "Movies - Bollywood"}
-    - {id: 65, cat: Movies/SD, desc: "Movies - CAM/TS"}
-    - {id: 80, cat: TV/Documentary, desc: "Documentary"}
-    - {id: 51, cat: Movies, desc: "Movies - Dubs/Dual Audio"}
-    - {id: 67, cat: Movies/DVD, desc: "Movies - DVDR"}
-    - {id: 1, cat: Movies/HD, desc: "Movies - HD 1080p"}
-    - {id: 2, cat: Movies/HD, desc: "Movies - HD 720p"}
-    - {id: 76, cat: Movies/HD, desc: "Movies - HEVC/x265"}
-    - {id: 47, cat: Movies/HD, desc: "Movies - X264/H264"}
-    - {id: 3, cat: Movies/UHD, desc: "Movies - UltraHD/4K"}
-    - {id: 42, cat: Movies/SD, desc: "Movies - XviD"}
-    - {id: 60, cat: Audio/Lossless, desc: "Music - FLAC"}
-    - {id: 59, cat: Audio/MP3, desc: "Music - MP3"}
-    - {id: 61, cat: Audio/Video, desc: "Music - Music Videos"}
-    - {id: 69, cat: Audio/Other, desc: "Music - Others"}
-    - {id: 95, cat: Other/Misc, desc: "Others - Misc"}
-    - {id: 78, cat: Other, desc: "Others - Unsorted"}
-    - {id: 63, cat: PC/Mobile-Android, desc: "Software - Android"}
-    - {id: 64, cat: PC/Mac, desc: "Software - Mac"}
-    - {id: 70, cat: PC, desc: "Software - Others"}
-    - {id: 62, cat: PC/0day, desc: "Software - Windows"}
-    - {id: 94, cat: Books, desc: "Tutorials - Tutorials"}
-    - {id: 79, cat: TV/Documentary, desc: "Documentary"}
-    - {id: 41, cat: TV/HD, desc: "TV - HD/X264/H264"}
-    - {id: 77, cat: TV/HD, desc: "TV - HEVC/x265"}
-    - {id: 5, cat: TV/SD, desc: "TV - SD/X264/H264"}
-    - {id: 50, cat: TV/SD, desc: "TV - SD/XVID"}
-    - {id: 72, cat: TV/Sport, desc: "TV - Sport"}
-    - {id: 7, cat: TV, desc: "TV - TV Packs"}
-    - {id: 89, cat: TV/UHD, desc: "TV - UltraHD/4K"}
-
-  modes:
-    search: [q]
-    tv-search: [q, season, ep]
-    movie-search: [q]
-    music-search: [q]
-    book-search: [q]
-
-settings:
-  - name: sort
-    type: select
-    label: Sort requested from site
-    default: id
-    options:
-      id: created
-      seeders: seeders
-      size: size
-      name: title
-  - name: type
-    type: select
-    label: Order requested from site
-    default: desc
-    options:
-      desc: desc
-      asc: asc
-
-download:
-  selectors:
-    - selector: a[href^="magnet:?xt="]
-      attribute: href
-
-search:
-  path: "{{ if .Keywords }}torrents-search.php{{ else }}torrents.php{{ end }}"
-  inputs:
-    $raw: "{{ range .Categories }}c{{.}}=1&{{end}}"
-    search: "{{ .Keywords }}"
-    sort: "{{ .Config.sort }}"
-    order: "{{ .Config.type }}"
-    incldead: 1
-  keywordsfilters:
-    - name: replace
-      args: ["-", " "] # remove special search character (negative search)
-    - name: replace
-      args: ["+", " "] # remove special search character (positive search)
-    - name: re_replace
-      args: ["(\\w+)", " +$1"] # prepend + to each word
-    - name: re_replace
-      args: ["\\s+", " "] # clean up whitespaces (multiple consecutive spaces confuses the search engine)
-
-  rows:
-    selector: div.myFrame-content > div > table > tbody > tr[class]
-    filters:
-      - name: andmatch # they ignore keywords <= characters, so we still andmatch it, see https://www.ettv.tv/forum/view/topic/and-search-operator-broken--70
-
-  fields:
-    download:
-      selector: a[href^="/torrent/"]
-      attribute: href
-    title:
-      selector: a[href^="/torrent/"][title]
-      attribute: title
-      filters:
-        - name: re_replace
-          args: ["( torrent)$", ""]
-    category:
-      text: 78 # issue #7110
-    category|noappend:
-      optional: true
-      selector: a[href^="/torrents.php?cat="]
-      attribute: href
-      filters:
-        - name: querystring
-          args: cat
-    details:
-      selector: a[href^="/torrent/"]
-      attribute: href
-    date:
-      selector: td:nth-child(3)
-    size:
-      selector: td:nth-child(4)
-      filters:
-        - name: replace
-          args: ["N/A", "0 B"]
-    seeders:
-      selector: td:nth-child(6)
-    leechers:
-      selector: td:nth-child(7)
-    downloadvolumefactor:
-      text: 0
-    uploadvolumefactor:
-      text: 1
-# engine n/a

src/Jackett.Common/Definitions/linkomanija.yml

@@ -49,6 +49,25 @@ caps:
     music-search: [q]
     book-search: [q]
 
+settings:
+  - name: username
+    type: text
+    label: Username
+
+  - name: password
+    type: password
+    label: Password
+
+#  - name: andmatch
+#    type: checkbox
+#    label: Accept non english characters in torrent names (get more results)
+#    default: true
+
+  - name: searchindesc
+    type: checkbox
+    label: Search in torrent description (get more results)
+    default: true
+
 login:
   path: takelogin.php
   method: post
@@ -67,11 +86,12 @@ search:
     $raw: "{{ range .Categories }}c{{.}}=1&{{end}}"
     search: "{{ .Keywords }}"
     incldead: 1
+    searchindesc: "{{ .Config.searchindesc }}"
 
   rows:
     selector: table> tbody > tr:has(a[href^="details?"])
-    filters:
-      - name: andmatch
+#    filters:
+#      - name: "{{ if .Config.andmatch }}andmatch{{ else }}{{ end }}" Currently, -name must be simple string, not a variable
 
   fields:
     category:

src/Jackett.Common/Definitions/pirateiro.yml

@@ -1,121 +0,0 @@
----
-id: pirateiro
-name: Pirateiro
-description: "Pirateiro is a Public site for MOVIES / TV / GENERAL"
-language: en-US
-type: public
-encoding: UTF-8
-followredirect: true
-links:
-  - https://pirateiro.com/
-  - https://pirateiro.eu/
-  - https://pirateiro.unblockit.how/
-legacylinks:
-  - http://pirateiro.com/
-  - https://pirateiro.unblockit.pw/
-  - https://pirateiro.unblockit.id/
-  - https://pirateiro.unblockit.win/
-  - https://pirateiro.unblockit.top/
-  - https://pirateiro.unblockit.lat/
-  - https://pirateiro.unblockit.app/
-  - https://pirateiro.unblockit.dev/
-  - https://pirateiro.unblockit.ltd/
-  - https://pirateiro.unblockit.link/
-  - https://pirateiro.unblockit.buzz/
-  - https://pirateiro.unblockit.club/
-  - https://pirateiro.unblockit.onl/
-  - https://pirateiro.unblockit.li/
-  - https://pirateiro.unblockit.uno/
-  - https://pirateiro.unblockit.ch/
-  - https://pirateiro.unblockit.ws/
-  - https://pirateiro.unblockit.kim/
-  - https://pirateiro.unblockit.bz/
-  - https://pirateiro.unblockit.tv/
-
-caps:
-  categorymappings:
-    - {id: cat100, cat: TV/Anime, desc: Anime}
-    - {id: cat200, cat: PC, desc: Software}
-    - {id: cat300, cat: Movies, desc: Movies}
-    - {id: cat400, cat: Console, desc: Games}
-    - {id: cat500, cat: Audio, desc: Music}
-    - {id: cat600, cat: Other, desc: Other}
-    - {id: cat700, cat: TV, desc: TV}
-    - {id: cat800, cat: XXX, desc: Adult}
-
-  modes:
-    search: [q]
-    tv-search: [q, season, ep]
-    movie-search: [q]
-    music-search: [q]
-
-settings:
-  - name: sort
-    type: select
-    label: Sort requested from site
-    default: enviado
-    options:
-      enviado: created
-      seeders: seeders
-      tamanho: size
-  - name: flaresolverr
-    type: info
-    label: FlareSolverr
-    default: This site may use Cloudflare DDoS Protection, therefore Jackett requires <a href="https://github.com/Jackett/Jackett#configuring-flaresolverr" target="_blank">FlareSolver</a> to access it.
-
-search:
-  paths:
-    - path: torrents/
-  inputs:
-    search: "{{ .Keywords }}"
-    orderby: "{{ .Config.sort }}"
-
-  rows:
-    selector: table.torrenttable tbody tr:has(a[href^="magnet:?xt="])
-
-  fields:
-    category:
-      selector: td:nth-child(1) a
-      attribute: class
-      filters:
-        # extract the first class
-        - name: split
-          args: [" ", 0]
-    title:
-      selector: td:nth-child(1) a:nth-child(2)
-    details:
-      selector: td:nth-child(1) a:nth-child(2)
-      attribute: href
-    download:
-      # .torrent urls can be missing
-      optional: true
-      selector: a[href$="/download"]
-      attribute: href
-    magnet:
-      selector: a[href^="magnet:?xt="]
-      attribute: href
-    date:
-      selector: td:nth-child(3) span
-      attribute: title
-      filters:
-        - name: append
-          args: " +01:00" # CET
-        - name: dateparse
-          args: "02/01/2006 15:04:05 -07:00"
-    size:
-      selector: td:nth-child(4)
-    seeders:
-      selector: td:nth-child(5)
-      filters:
-        - name: replace
-          args: [" ", ""]
-    leechers:
-      selector: td:nth-child(6)
-      filters:
-        - name: replace
-          args: [" ", ""]
-    downloadvolumefactor:
-      text: 0
-    uploadvolumefactor:
-      text: 1
-# engine n/a

src/Jackett.Common/Indexers/NewPCT.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
 using System.Globalization;
 using System.Linq;
+using System.Net;
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
@@ -14,6 +15,7 @@ using Jackett.Common.Utils.Clients;
 using Newtonsoft.Json.Linq;
 using NLog;
 using static Jackett.Common.Models.IndexerConfig.ConfigurationData;
+using WebClient = Jackett.Common.Utils.Clients.WebClient;
 
 namespace Jackett.Common.Indexers
 {
@@ -53,12 +55,6 @@ namespace Jackett.Common.Indexers
             public override object Clone() => new NewpctRelease(this);
         }
 
-        private class DownloadMatcher
-        {
-            public Regex MatchRegex;
-            public MatchEvaluator MatchEvaluator;
-        }
-
         private readonly char[] _wordSeparators = { ' ', '.', ',', ';', '(', ')', '[', ']', '-', '_' };
         private readonly int _wordNotFoundScore = 100000;
         private readonly Regex _searchStringRegex = new Regex(@"(.+?)S(\d{2})(E(\d{2}))?$", RegexOptions.IgnoreCase);
@@ -67,18 +63,6 @@ namespace Jackett.Common.Indexers
         // Love 101 - Temp. 1 Capitulos 1 al 8
         private readonly Regex _seriesChaptersTitleRegex = new Regex(@"(.+)Temp. (\d+) Capitulos (\d+) al (\d+)", RegexOptions.IgnoreCase);
         private readonly Regex _titleYearRegex = new Regex(@" *[\[\(]? *((19|20)\d{2}) *[\]\)]? *$");
-        private readonly DownloadMatcher[] _downloadMatchers =
-        {
-            new DownloadMatcher
-            {
-                MatchRegex = new Regex("(/descargar-torrent/[^\"]+)\"")
-            },
-            new DownloadMatcher
-            {
-                MatchRegex = new Regex(@"window\.location\.href\s*=\s*""([^""]+)"""),
-                MatchEvaluator = m => $"https:{m.Groups[1]}"
-            }
-        };
 
         private readonly int _maxMoviesPages = 6;
         private readonly int[] _allTvCategories = (new[] { TorznabCatType.TV }).Concat(TorznabCatType.TV.SubCategories).Select(c => c.ID).ToArray();
@@ -188,46 +172,33 @@ namespace Jackett.Common.Indexers
 
         public override async Task<byte[]> Download(Uri linkParam)
         {
+            Uri uriLink = null;
             var downloadLink = linkParam.AbsoluteUri.Replace("/descargar/", "/descargar/torrent/");
+            var result = await RequestWithCookiesAndRetryAsync(downloadLink, referer: linkParam.AbsoluteUri);
+
+            var downloadRegex = new Regex("/t_download/([0-9]+)/");
+            var match = downloadRegex.Match(result.ContentString);
+            if (match.Success)
+            {
+                const string downloadUrl = "https://atomtt.com/to.php";
+                var headers = new Dictionary<string, string>
+                {
+                    {"X-Requested-With", "XMLHttpRequest"},
+                    {"Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"}
+                };
+                var body = "t=" + match.Groups[1].Value;
+                result = await RequestWithCookiesAsync(downloadUrl, method: RequestType.POST, rawbody: body,
+                                                       headers: headers, referer: downloadLink);
+                if (result.Status == HttpStatusCode.OK)
+                    uriLink = new Uri(SiteLink + "t_download/" + result.ContentString + ".torrent");
+            }
 
-            var results = await RequestWithCookiesAndRetryAsync(downloadLink, referer: linkParam.AbsoluteUri);
-            var uriLink = ExtractDownloadUri(results.ContentString, downloadLink);
             if (uriLink == null)
                 throw new Exception("Download link not found!");
 
             return await base.Download(uriLink);
         }
 
-        private Uri ExtractDownloadUri(string content, string baseLink)
-        {
-            foreach (var matcher in _downloadMatchers)
-            {
-                var match = matcher.MatchRegex.Match(content);
-                if (match.Success)
-                {
-                    string linkText;
-
-                    if (matcher.MatchEvaluator != null)
-                        linkText = (string)matcher.MatchEvaluator.DynamicInvoke(match);
-                    else
-                        linkText = match.Groups[1].Value;
-
-                    // take the details page link and the download page link and build a Torrent link
-                    // Details page: https://atomixhq.com/descargar/torrent/peliculas-x264-mkv/el-viaje-i-onde-dager--2021-/bluray-microhd/
-                    // Download page: https://atomtt.com/t_download/159843_-1634325135-El-viaje--I-onde-dager---2021---BluRay-MicroHD/
-                    // Torrent link: https://atomixhq.com/t_download/159843_-1634325135-El-viaje--I-onde-dager---2021---BluRay-MicroHD.torrent
-                    linkText = linkText.Remove(linkText.Length - 1, 1) + ".torrent";
-                    var linkHost = new Uri(linkText).Host;
-                    var linkBase = new Uri(baseLink).Host;
-                    var downloadLink = linkText.Replace(linkHost.ToString(), linkBase.ToString());
-
-                    return new Uri(downloadLink);
-                }
-            }
-
-            return null;
-        }
-
         protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
         {
             var releases = new List<ReleaseInfo>();

src/Jackett.Updater/Program.cs

@@ -328,6 +328,7 @@ namespace Jackett.Updater
                 "Definitions/erzsebetpl.yml",
                 "Definitions/estrenosdtl.yml",
                 "Definitions/ethor.yml",
+                "Definitions/ettv.yml",
                 "Definitions/evolutionpalace.yml",
                 "Definitions/exoticaz.yml", // migrated to C#
                 "Definitions/extratorrent-ag.yml",
@@ -417,6 +418,7 @@ namespace Jackett.Updater
                 "Definitions/obscure.yml",
                 "Definitions/oncesearch.yml",
                 "Definitions/passionetorrent.yml",
+                "Definitions/pirateiro.yml",
                 "Definitions/pleasuredome.yml",
                 "Definitions/polishtracker.yml",
                 "Definitions/pornorip.yml",