cardigann: check for page size

https://github.com/Prowlarr/Indexers/pull/338

README.md

@@ -388,7 +388,6 @@ A third-party Golang SDK for Jackett is available from [webtor-io/go-jackett](ht
  * HDVIDEO
  * HDZone
  * Hebits
- * HellasHut
  * HHanClub
  * House of Devil
  * HQMusic

src/Jackett.Common/Definitions/1337x.yml

@@ -43,6 +43,8 @@ legacylinks:
   - https://1337x.unblockit.click/
 
 caps:
+  limitsDefault: 80
+  limitsMax: 80
   categorymappings:
     # Anime
     - {id: 28, cat: TV/Anime, desc: "Anime/Anime"}
@@ -181,6 +183,9 @@ download:
       attribute: href
 
 search:
+  pageSize: 20
+  pageable: "{{ if or .Query.Album .Query.Artist .Keywords }}true{{ else }}false{{ end }}"
+
   paths:
     # present first page of movies tv and music results if there are no search parms supplied (20 hits per page)
     - path: "{{ if or .Query.Album .Query.Artist .Keywords }}sort-search{{ else }}cat/Movies{{ end }}{{ if or .Query.Album .Query.Artist }}/{{ or .Query.Album .Query.Artist }}{{ else }}/{{ .Keywords }}{{ end }}{{ if or .Query.Album .Query.Artist .Keywords }}/{{ else }}{{ end }}{{ .Config.sort }}/{{ .Config.type }}/1/"

src/Jackett.Common/Definitions/bitsearch.yml

@@ -9,6 +9,7 @@ requestDelay: 2
 links:
   - https://bitsearch.to/
   - https://bitsearch.mrunblock.life/
+  - https://bitsearch.nocensor.click/
 legacylinks:
   - https://bitsearch.nocensor.biz/
   - https://bitsearch.nocensor.sbs/

src/Jackett.Common/Definitions/fantastiko.yml

@@ -6,7 +6,9 @@ language: el-GR
 type: private
 encoding: UTF-8
 links:
-  - http://fantastiko.space/ # https is not supported
+  - https://fantastiko.space/
+legacylinks:
+  - http://fantastiko.space/
 
 caps:
   categorymappings:

src/Jackett.Common/Definitions/gimmepeers.yml

@@ -96,8 +96,8 @@ search:
   inputs:
     $raw: "{{ range .Categories }}c{{.}}=1&{{end}}"
     search: "{{ if .Query.IMDBID }}{{ .Query.IMDBID }}{{ else }}{{ .Keywords }}{{ end }}{{ if .Query.Genre }} genre:{{ .Query.Genre }}{{ else }}{{ end }}"
-    # 0 title, 1 nfo, 2 filelist, 3 title+nfo
-    blah: "{{ if or .Query.IMDBID .Query.Genre }}3{{ else }}0{{ end }}"
+    # 0 title, 1 nfo, 2 filelist, 3 title+nfo, 4 imdb
+    blah: "{{ if .Query.IMDBID }}4{{ else }}{{ end }}{{ if .Query.Genre }}3{{ else }}{{ end }}{{ if or .Query.IMDBID .Query.Genre }}{{ else }}0{{ end }}"
     # 0 active only, 1 dead only
     incldead: 0
     sort: "{{ .Config.sort }}"

src/Jackett.Common/Definitions/gktorrent.yml

@@ -9,6 +9,7 @@ followredirect: true
 # to fetch current domain use https://www.protege-liens.com/Gktorrent
 links:
   - https://www.gktorrents.cc/
+  - https://gktorrent.nocensor.click/
 legacylinks:
   - https://www.gktorrent.org/
   - https://www.gktorrent.me/

src/Jackett.Common/Definitions/hdcztorrent.yml

@@ -122,7 +122,7 @@ search:
     by: "{{ .Config.type }}"
 
   rows:
-    selector: td#mcol table.lista > tbody > tr:has(a[href*=bookmark])
+    selector: td#mcol table.lista > tbody > tr:has(a[href^="magnet:?xt="])
 
   fields:
     category:
@@ -132,13 +132,16 @@ search:
         - name: querystring
           args: category
     title:
-      selector: td:nth-child(2) > a
+      selector: a[href^="index.php?page=torrent-details"]
     details:
       selector: a[href^="index.php?page=torrent-details"]
       attribute: href
     download:
       selector: a[href^="download.php"]
       attribute: href
+    magnet:
+      selector: a[href^="magnet:?xt="]
+      attribute: href
     poster:
       selector: a[href^="index.php?page=torrent-details"]
       attribute: onmouseover
@@ -146,17 +149,19 @@ search:
         - name: regexp
           args: "src=(.+?) "
     date:
-      selector: td:nth-child(6)
+      selector: td:nth-child(7)
       # auto adjusted by site account profile
       filters:
         - name: dateparse
           args: "dd/MM/yyyy"
     size:
-      selector: td:nth-child(11)
+      selector: td:nth-child(12)
     seeders:
-      selector: td:nth-child(8)
-    leechers:
       selector: td:nth-child(9)
+    leechers:
+      selector: td:nth-child(10)
+    grabs:
+      selector: td:nth-child(11)
     downloadvolumefactor:
       case:
         img[src$="freeleech.gif"]: 0

src/Jackett.Common/Definitions/hellashut.yml

@@ -1,193 +0,0 @@
----
-id: hellashut
-name: HellasHut
-description: "HellasHut is a GREEK Private Torrent Tracker for MOVIES / TV / GENERAL"
-language: el-GR
-type: private
-encoding: UTF-8
-testlinktorrent: false
-links:
-  - https://hellashut.de/
-
-caps:
-  categorymappings:
-    - {id: 28, cat: TV/Anime, desc: "Anime/Cartoon HD"}
-    - {id: 47, cat: TV/Anime, desc: "Anime/Cartoon HDTV"}
-    - {id: 48, cat: TV/Anime, desc: "Anime/Cartoon SD"}
-    - {id: 50, cat: TV/Anime, desc: "Anime/Cartoon SD Emb"}
-    - {id: 51, cat: TV/Anime, desc: "Anime/Cartoon TV"}
-    - {id: 77, cat: PC, desc: "Applications DOS"}
-    - {id: 20, cat: PC, desc: "Applications Linux"}
-    - {id: 19, cat: PC/Mac, desc: "Applications MAC"}
-    - {id: 68, cat: PC, desc: "Applications OS2 WARP"}
-    - {id: 21, cat: PC, desc: "Applications Other"}
-    - {id: 18, cat: PC/0day, desc: "Applications Windows"}
-    - {id: 52, cat: Books, desc: "Books/Magazines All"}
-    - {id: 9, cat: TV/Documentary, desc: "Documentaries All"}
-    - {id: 53, cat: TV/Documentary, desc: "Documentaries Greek"}
-    - {id: 54, cat: TV/Documentary, desc: "Documentaries HD"}
-    - {id: 55, cat: Movies/DVD, desc: "DVD-R All"}
-    - {id: 17, cat: Console/Other, desc: "Games Other"}
-    - {id: 11, cat: Console/PS4, desc: "Games PSX"}
-    - {id: 10, cat: PC/Games, desc: "Games Windows"}
-    - {id: 13, cat: Console/XBox, desc: "Games Xbox"}
-    - {id: 56, cat: Other, desc: "Miscellaneous All"}
-    - {id: 57, cat: PC/Mobile-Other, desc: "Mobile All"}
-    - {id: 1, cat: Movies/3D, desc: "Movies 3D"}
-    - {id: 2, cat: Movies/UHD, desc: "Movies 4K"}
-    - {id: 42, cat: Movies/BluRay, desc: "Movies BluRay"}
-    - {id: 58, cat: Movies/DVD, desc: "Movies DVD"}
-    - {id: 4, cat: Movies/HD, desc: "Movies HD"}
-    - {id: 69, cat: Movies/HD, desc: "Movies HD Packs"}
-    - {id: 3, cat: Movies/SD, desc: "Movies SD"}
-    - {id: 67, cat: Movies/SD, desc: "Movies SD Packs"}
-    - {id: 71, cat: Audio, desc: "Music CDA"}
-    - {id: 59, cat: Audio, desc: "Music Discographies"}
-    - {id: 23, cat: Audio/Lossless, desc: "Music FLAC"}
-    - {id: 22, cat: Audio/MP3, desc: "Music MP3"}
-    - {id: 25, cat: Audio/Video, desc: "Music Video"}
-    - {id: 75, cat: Movies/HD, desc: "NO IN SUBS Movies HD"}
-    - {id: 76, cat: Movies/UHD, desc: "NO SUBS IN Movies 4K"}
-    - {id: 35, cat: TV/Sport, desc: "Sports HD"}
-    - {id: 34, cat: TV/Sport, desc: "Sports SD"}
-    - {id: 5, cat: TV, desc: "TV Greek Movies"}
-    - {id: 6, cat: TV, desc: "TV Greek Series"}
-    - {id: 7, cat: TV, desc: "TV Greek Series Packs"}
-    - {id: 60, cat: TV, desc: "TV Greek Shows"}
-    - {id: 41, cat: TV/HD, desc: "TV HD"}
-    - {id: 61, cat: TV, desc: "TV Series"}
-    - {id: 62, cat: TV, desc: "TV Series Emb"}
-    - {id: 63, cat: TV, desc: "TV Series Emb Packs"}
-    - {id: 64, cat: TV/HD, desc: "TV Series HD"}
-    - {id: 65, cat: TV/HD, desc: "TV Series HD Packs"}
-    - {id: 66, cat: TV, desc: "TV Series Packs"}
-    - {id: 73, cat: Movies/BluRay, desc: "UNTOUCHED BluRay"}
-    - {id: 72, cat: Movies/DVD, desc: "UNTOUCHED DVD"}
-
-  modes:
-    search: [q]
-    tv-search: [q, season, ep, imdbid]
-    movie-search: [q, imdbid]
-    music-search: [q]
-    book-search: [q]
-
-settings:
-  - name: username
-    type: text
-    label: Username
-  - name: password
-    type: password
-    label: Password
-  - name: freeleech
-    type: checkbox
-    label: Search freeleech only
-    default: false
-  - name: sort
-    type: select
-    label: Sort requested from site
-    default: id
-    options:
-      id: created
-      seeders: seeders
-      size: size
-      name: title
-  - name: type
-    type: select
-    label: Order requested from site
-    default: desc
-    options:
-      desc: desc
-      asc: asc
-
-login:
-  path: account-login.php
-  method: post
-  inputs:
-    username: "{{ .Config.username }}"
-    password: "{{ .Config.password }}"
-    returnto: /
-  error:
-    - selector: td.embedded:contains("Access Denied")
-  test:
-    path: /
-    selector: a[href="account-logout.php"]
-
-download:
-  selectors:
-    - selector: a[href^="download.php?id="], a[href^="magnet:?xt="]
-      attribute: href
-
-search:
-  paths:
-    # https://hellashut.de/torrents-search.php?c52=1&search=&cat=0&incldead=1&advanced=1&lang=0&blah=0&freeleech=0&sort=id&order=desc
-    - path: torrents-search.php
-  inputs:
-    $raw: "{{ range .Categories }}c{{.}}=1&{{end}}"
-    search: "{{ if .Query.IMDBID }}{{ .Query.IMDBID }}{{ else }}{{ .Keywords }}{{ end }}"
-    # 0 active, 1 incldead, 2 onlydead
-    incldead: 1
-    # 0 all, 1 notfree, 2 onlyfree
-    freeleech: "{{ if .Config.freeleech }}2{{ else }}0{{ end }}"
-    # 0 all, 1 English, 2 French, 3 German, 4 Italian, 5 Japanese, 6 Spanish, 7 Russian, 8 Greek
-    lang: 0
-    # 0 name, 1 descr, 2 uploader
-    blah: "{{ if .Query.IMDBID }}1{{ else }}0{{ end }}"
-    # 0 all
-    team: 0
-    sort: "{{ .Config.sort }}"
-    order: "{{ .Config.type }}"
-    # does not return imdb link in results
-
-  rows:
-    selector: table.ttable_headinner tr.t-row
-
-  fields:
-    category:
-      selector: a[href*="cat="]
-      attribute: href
-      optional: true
-      default: 56
-      filters:
-        - name: querystring
-          args: cat
-    title:
-      selector: a[href^="torrents-details.php?id="]
-      attribute: title
-    details:
-      selector: a[href^="torrents-details.php?id="]
-      attribute: href
-      filters:
-        - name: replace
-          args: ["&hit=1", ""] # avoid redirect
-    download:
-      selector: a[href^="torrents-details.php?id="]
-      attribute: href
-      filters:
-        - name: replace
-          args: ["&hit=1", ""] # avoid redirect
-    size:
-      selector: td:nth-last-child(7)
-    date:
-      selector: td:nth-last-child(6)
-      # auto adjusted by site account profile
-      filters:
-        - name: dateparse
-          args: "dd-MM-yyyy HH:mm:ss"
-    grabs:
-      selector: td:nth-last-child(5)
-    seeders:
-      selector: td:nth-last-child(4)
-    leechers:
-      selector: td:nth-last-child(3)
-    downloadvolumefactor:
-      case:
-        img[src="pic/freedownload.gif"]: 0
-        "*": 1
-    uploadvolumefactor:
-      text: 1
-    minimumratio:
-      text: 1.0
-    minimumseedtime:
-      # 10 day (as seconds = 10 x 24 x 60 x 60)
-      text: 864000
-# TorrentTrader v2-svn

src/Jackett.Common/Definitions/movietorrent.yml

@@ -10,6 +10,8 @@ links:
   - https://movietorrent.co/
 
 caps:
+  limitsDefault: 36
+  limitsMax: 36
   categorymappings:
     - {id: 1, cat: Movies, desc: "Bollywood"}
     - {id: 2, cat: Movies/HD, desc: "1080p"}
@@ -74,6 +76,9 @@ download:
         - name: validfilename
 
 search:
+  pageSize: 12
+  pageable: true
+
   paths:
     - path: "?s={{ .Keywords }}"
     - path: "/page/2/?s={{ .Keywords }}"

src/Jackett.Common/Definitions/oldtoonsworld.yml

@@ -86,6 +86,7 @@ login:
   error:
     - selector: td.embedded:has(h2:contains("Login failed!"))
     - selector: td.embedded:has(h2:contains("Failed"))
+    - selector: td.embedded:has(h2:contains("Login Locked!"))
   test:
     path: index.php
     selector: a[href="logout.php"]

src/Jackett.Common/Definitions/schema.json

@@ -93,6 +93,14 @@
             "type": "object",
             "additionalProperties": false,
             "properties": {
+                "limitsDefault": {
+                    "type": "integer",
+                    "minimum": 1
+                },
+                "limitsMax": {
+                    "type": "integer",
+                    "minimum": 1
+                },
                 "categories": {
                     "type": "object",
                     "additionalProperties": false,
@@ -482,6 +490,20 @@
             "type": "object",
             "additionalProperties": false,
             "properties": {
+                "pageSize": {
+                    "type": "integer",
+                    "minimum": 1
+                },
+                "pageable": {
+                    "oneOf": [
+                        {
+                            "type": "boolean"
+                        },
+                        {
+                            "type": "string"
+                        }
+                    ]
+                },
                 "path": {
                     "type": "string"
                 },

src/Jackett.Common/Definitions/solidtorrents.yml

@@ -8,6 +8,7 @@ encoding: UTF-8
 requestDelay: 2
 links:
   - https://solidtorrents.to/
+  - https://solidtorrents.nocensor.click/
 legacylinks:
   - https://solidtorrents.net/
 

src/Jackett.Common/Definitions/torlock.yml

@@ -12,6 +12,7 @@ links:
   - https://www.torlock.com/
   - https://torlock.unblockit.asia/
   - https://torlock.mrunblock.life/
+  - https://torlock.nocensor.click/
 legacylinks:
   - https://torlock.com/
   - https://torlock.unblockit.cam/

src/Jackett.Common/Definitions/torrentdownload.yml

@@ -10,6 +10,7 @@ links:
   - https://www.torrentdownload.info/
   - https://torrentdownload.unblockit.asia/
   - https://torrentdownload.mrunblock.life/
+  - https://torrentdownload.nocensor.click/
 legacylinks:
   - https://torrentdownload.nocensor.space/
   - https://torrentdownload.nocensor.work/

src/Jackett.Common/Definitions/torrentfunk.yml

@@ -11,6 +11,7 @@ links:
   - https://www.torrentfunk2.com/
   - https://torrentfunk.unblockit.asia/
   - https://torrentfunk.mrunblock.life/
+  - https://torrentfunk.nocensor.click/
 legacylinks:
   - https://torrentfunk.nocensor.space/
   - https://torrentfunk.nocensor.work/

src/Jackett.Common/Definitions/torrentsir.yml

@@ -7,7 +7,7 @@ type: public
 encoding: UTF-8
 followredirect: true
 links:
-  - https://torrentsir104.com/
+  - https://torrentsir105.com/
 legacylinks:
   - https://torrentsir90.com/
   - http://torrentsir91.com/
@@ -31,6 +31,8 @@ legacylinks:
   - http://torrentsir103.com/
   - https://torrentsir103.com/
   - http://torrentsir104.com/
+  - https://torrentsir104.com/
+  - http://torrentsir105.com/
 
 caps:
   categorymappings:

src/Jackett.Common/Definitions/yourbittorrent.yml

@@ -9,6 +9,7 @@ links:
   - https://yourbittorrent.com/
   - https://yourbittorrent2.com/
   - https://yourbittorrent.mrunblock.life/
+  - https://yourbittorrent.nocensor.click/
 legacylinks:
   - https://yourbittorrent.host/
   - https://yourbittorrent.nocensor.space/

src/Jackett.Common/Indexers/BaseIndexer.cs

@@ -35,6 +35,7 @@ namespace Jackett.Common.Indexers
         public virtual string Type { get; protected set; }
 
         public virtual bool SupportsPagination => false;
+        public virtual int PageSize => 0;
 
         public virtual bool IsConfigured { get; protected set; }
         public virtual string[] Tags { get; protected set; }

src/Jackett.Common/Indexers/CardigannIndexer.cs

@@ -27,6 +27,8 @@ namespace Jackett.Common.Indexers
 {
     public class CardigannIndexer : BaseWebIndexer
     {
+        public override int PageSize => Definition.Search != null && Definition.Search.PageSize > 0 ? Definition.Search.PageSize : 1;
+
         protected IndexerDefinition Definition;
         protected WebResult landingResult;
         protected IHtmlDocument landingResultDocument;
@@ -124,6 +126,8 @@ namespace Jackett.Common.Indexers
             TorznabCaps = new TorznabCapabilities();
             TorznabCaps.ParseCardigannSearchModes(Definition.Caps.Modes);
             TorznabCaps.SupportsRawSearch = Definition.Caps.Allowrawsearch;
+            TorznabCaps.LimitsDefault = Definition.Caps.LimitsDefault ?? TorznabCaps.LimitsDefault;
+            TorznabCaps.LimitsMax = Definition.Caps.LimitsMax ?? TorznabCaps.LimitsMax;
 
             // init config Data
             configData = new ConfigurationData();
@@ -1353,6 +1357,13 @@ namespace Jackett.Common.Indexers
             variables[".Query.Keywords"] = string.Join(" ", KeywordTokens);
             variables[".Keywords"] = applyFilters((string)variables[".Query.Keywords"], Search.Keywordsfilters, variables);
 
+            var pageSize = PageSize;
+
+            if (!bool.TryParse(applyGoTemplateText(Search.Pageable, variables), out var pageable))
+            {
+                pageable = false;
+            }
+
             // TODO: prepare queries first and then send them parallel
             var SearchPaths = Search.Paths;
             foreach (var SearchPath in SearchPaths)
@@ -1734,14 +1745,28 @@ namespace Jackett.Common.Indexers
                         OnParseError(results, ex);
                     }
                 }
+
+                pageSize = pageSize == 1 ? releases.Count : pageSize;
+
+                if (pageable && !IsFullPage(releases, pageSize))
+                {
+                    break;
+                }
             }
 
             if (query.Limit > 0)
+            {
                 releases = releases.Take(query.Limit).ToList();
+            }
 
             return releases;
         }
 
+        protected virtual bool IsFullPage(IList<ReleaseInfo> page, int pageSize)
+        {
+            return pageSize != 0 && page.Count >= pageSize;
+        }
+
         protected async Task<WebResult> handleRequest(requestBlock request, Dictionary<string, object> variables = null, string referer = null)
         {
             var requestLinkStr = resolvePath(applyGoTemplateText(request.Path, variables)).ToString();

src/Jackett.Common/Indexers/DonTorrent.cs

@@ -25,10 +25,10 @@ namespace Jackett.Common.Indexers
         public override string Id => "dontorrent";
         public override string Name => "DonTorrent";
         public override string Description => "DonTorrent is a SPANISH public tracker for MOVIES / TV / GENERAL";
-        public override string SiteLink { get; protected set; } = "https://dontorrent.company/";
+        public override string SiteLink { get; protected set; } = "https://dontorrent.discount/";
         public override string[] AlternativeSiteLinks => new[]
         {
-            "https://dontorrent.company/",
+            "https://dontorrent.discount/",
             "https://todotorrents.net/",
             "https://tomadivx.net/",
             "https://seriesblanco.one/",
@@ -37,7 +37,6 @@ namespace Jackett.Common.Indexers
         };
         public override string[] LegacySiteLinks => new[]
         {
-            "https://dontorrent.beer/",
             "https://dontorrent.surf/",
             "https://dontorrent.how/",
             "https://dontorrent.casa/",
@@ -51,7 +50,8 @@ namespace Jackett.Common.Indexers
             "https://dontorrent.ms/",
             "https://dontorrent.care/",
             "https://dontorrent.cash/",
-            "https://dontorrent.observer/"
+            "https://dontorrent.observer/",
+            "https://dontorrent.company/"
 
         };
         public override string Language => "es-ES";

src/Jackett.Common/Indexers/EraiRaws.cs

@@ -24,7 +24,8 @@ namespace Jackett.Common.Indexers
         {
             "https://www.erai-raws.info/",
             "https://beta.erai-raws.info/",
-            "https://erairaws.mrunblock.guru/"
+            "https://erairaws.mrunblock.life/",
+            "https://erairaws.nocensor.click/"
         };
         public override string[] LegacySiteLinks => new[]
         {
@@ -34,7 +35,8 @@ namespace Jackett.Common.Indexers
             "https://erairaws.nocensor.sbs/",
             "https://erairaws.nocensor.world/",
             "https://erairaws.nocensor.lol/",
-            "https://erairaws.nocensor.art/"
+            "https://erairaws.nocensor.art/",
+            "https://erairaws.mrunblock.guru/"
         };
         public override string Language => "en-US";
         public override string Type => "semi-private";

src/Jackett.Common/Indexers/SubsPlease.cs

@@ -26,7 +26,8 @@ namespace Jackett.Common.Indexers
         public override string[] AlternativeSiteLinks => new[]
         {
             "https://subsplease.org/",
-            "https://subsplease.mrunblock.guru/"
+            "https://subsplease.mrunblock.life/",
+            "https://subsplease.nocensor.click/"
         };
         public override string[] LegacySiteLinks => new[]
         {
@@ -36,7 +37,8 @@ namespace Jackett.Common.Indexers
             "https://subsplease.nocensor.sbs/",
             "https://subsplease.nocensor.world/",
             "https://subsplease.nocensor.lol/",
-            "https://subsplease.nocensor.art/"
+            "https://subsplease.nocensor.art/",
+            "https://subsplease.mrunblock.guru/"
         };
         public override string Language => "en-US";
         public override string Type => "public";

src/Jackett.Common/Models/IndexerDefinition.cs

@@ -69,6 +69,8 @@ namespace Jackett.Common.Models
 
     public class capabilitiesBlock
     {
+        public int? LimitsMax { get; set; }
+        public int? LimitsDefault { get; set; }
         public Dictionary<string, string> Categories { get; set; }
         public List<CategorymappingBlock> Categorymappings { get; set; }
         public Dictionary<string, List<string>> Modes { get; set; }
@@ -137,6 +139,8 @@ namespace Jackett.Common.Models
 
     public class searchBlock
     {
+        public int PageSize { get; set; }
+        public string Pageable { get; set; }
         public string Path { get; set; }
         public List<searchPathBlock> Paths { get; set; }
         public Dictionary<string, List<string>> Headers { get; set; }

src/Jackett.Updater/Program.cs

@@ -401,6 +401,7 @@ namespace Jackett.Updater
                 "Definitions/hdreactor.yml", // renamed to hdhouse
                 "Definitions/hdspain.yml",
                 "Definitions/hdstreet.yml",
+                "Definitions/hellashut.yml",
                 "Definitions/hellastz.yml",
                 "Definitions/hidden-palace.yml",
                 "Definitions/hon3yhd-net.yml",