retroflix: migrate the indexer to c# and use the new api. resolves #10838 (#10847)

src/Jackett.Common/Definitions/retroflix.yml

@@ -1,181 +0,0 @@
----
-id: retroflix
-name: RetroFlix
-description: "Private Torrent Tracker for Classic Movies / TV / General Releases."
-language: en-us
-type: private
-encoding: UTF-8
-links:
-  - https://retroflix.club/
-legacylinks:
-  - https://retroflix.net/
-
-caps:
-  categorymappings:
-    - {id: 401, cat: Movies, desc: "Movies"}
-    - {id: 402, cat: TV, desc: "TV Series"}
-    - {id: 406, cat: Audio/Video, desc: "Music Videos"}
-    - {id: 407, cat: TV/Sport, desc: "Sports"}
-    - {id: 409, cat: Books, desc: "Books"}
-    - {id: 408, cat: Audio, desc: "HQ Audio"}
-
-  modes:
-    search: [q]
-    tv-search: [q, season, ep, imdbid]
-    movie-search: [q, imdbid]
-    music-search: [q]
-    book-search: [q]
-
-settings:
-  - name: cookie
-    type: text
-    label: Cookie
-  - name: info
-    type: info
-    label: How to get the Cookie
-    default: "<ol><li>Login to this tracker with your browser<li>Open the <b>DevTools</b> panel by pressing <b>F12</b><li>Select the <b>Network</b> tab<li>Click on the <b>Doc</b> button (Chrome Browser) or <b>HTML</b> button (FireFox)<li>Refresh the page by pressing <b>F5</b><li>Click on the first row entry<li>Select the <b>Headers</b> tab on the Right panel<li>Find <b>'cookie:'</b> in the <b>Request Headers</b> section<li><b>Select</b> and <b>Copy</b> the whole cookie string <i>(everything after 'cookie: ')</i> and <b>Paste</b> here.</ol>"
-  - name: freeleech
-    type: checkbox
-    label: Search freeleech only
-    default: false
-  - name: sort
-    type: select
-    label: Sort requested from site
-    default: 4
-    options:
-      4: created
-      7: seeders
-      5: size
-      1: title
-  - name: type
-    type: select
-    label: Order requested from site
-    default: desc
-    options:
-      desc: desc
-      asc: asc
-  - name: info_tpp
-    type: info
-    label: Results Per Page
-    default: For best results, change the <b>Torrents per page:</b> setting to <b>100</b> on your account profile.
-
-login:
-  method: cookie
-  inputs:
-    cookie: "{{ .Config.cookie }}"
-  test:
-    path: torrents.php
-    selector: a[href*="/logout?"]
-
-search:
-  # https://retroflix.club/torrents.php?incldead=0&spstate=0&inclbookmarked=0&search=tt0055254&search_area=4&search_mode=0
-  paths:
-    - path: torrents.php
-  inputs:
-    $raw: "{{ range .Categories }}cat{{.}}=1&{{end}}"
-    search: "{{ if .Query.IMDBID }}{{ .Query.IMDBID }}{{ else }}{{ .Keywords }}{{ end }}"
-    # 0 incldead, 1 active, 2 dead
-    incldead: 0
-    # 0 all, 1 normal, 2 free, 3 2x, 4 2xfree, 5 50%, 6 2x50%, 7 30%
-    spstate: "{{ if .Config.freeleech }}2{{ else }}0{{ end }}"
-    # 0 title, 1 descr, 3 uploader, 4 imdburl
-    search_area: "{{ if .Query.IMDBID }}4{{ else }}0{{ end }}"
-    # 0 AND, 1 OR, 2 Exact
-    search_mode: 0
-    sort: "{{ .Config.sort }}"
-    type: "{{ .Config.type }}"
-
-  rows:
-    selector: table.torrents > tbody > tr:has(table.torrentname)
-
-  fields:
-    category:
-      selector: a[href^="?cat="]
-      attribute: href
-      filters:
-        - name: querystring
-          args: cat
-    release_year:
-      selector: a[href^="/torrents.php?processing="]
-      optional: true
-    quality:
-      selector: a[href^="/torrents.php?standard="]
-      optional: true
-    title:
-      selector: a[href^="details.php?id="]
-      filters:
-        - name: append
-          args: " {{ .Result.release_year }}"
-        - name: append
-          args: " {{ .Result.quality }}"
-    title:
-      selector: a[title][href^="details.php?id="]
-      attribute: title
-      optional: true
-      filters:
-        - name: append
-          args: " {{ .Result.release_year }}"
-        - name: append
-          args: " {{ .Result.quality }}"
-    details:
-      selector: a[href^="details.php?id="]
-      attribute: href
-    download:
-      selector: a[href^="download.php?id="]
-      attribute: href
-    poster:
-      selector: tr[onmouseover]
-      attribute: onmouseover
-      filters:
-        - name: regexp
-          args: "src=(.+?) "
-    imdb:
-      selector: a[href*="imdb.com/title/tt"]
-      attribute: href
-    date:
-      # time type: time elapsed (default)
-      selector: td:nth-child(4) > span[title]
-      attribute: title
-      optional: true
-      filters:
-        - name: append
-          args: " +00:00" # auto adjusted by site account profile
-        - name: dateparse
-          args: "02-01-2006 15:04:05 -07:00"
-    date:
-      # time added
-      selector: td:nth-child(4):not(:has(span))
-      optional: true
-      filters:
-        - name: append
-          args: " +00:00" # auto adjusted by site account profile
-        - name: dateparse
-          args: "02-01-200615:04:05 -07:00"
-    size:
-      selector: td:nth-child(5)
-    seeders:
-      selector: td:nth-child(6)
-    leechers:
-      selector: td:nth-child(7)
-    grabs:
-      selector: td:nth-child(8)
-    downloadvolumefactor:
-      case:
-        img.pro_free: 0
-        img.pro_free2up: 0
-        img.pro_50pctdown: 0.5
-        img.pro_50pctdown2up: 0.5
-        img.pro_30pctdown: 0.3
-        "*": 1
-    uploadvolumefactor:
-      case:
-        img.pro_50pctdown2up: 2
-        img.pro_free2up: 2
-        img.pro_2up: 2
-        "*": 1
-    minimumratio:
-      text: 1.0
-    minimumseedtime:
-      # 3 days (as seconds = 3 x 24 x 60 x 60)
-      text: 259200
-# NexusPHP

src/Jackett.Common/Indexers/Abstract/SpeedAppTracker.cs

@@ -0,0 +1,184 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Globalization;
+using System.Linq;
+using System.Net;
+using System.Threading.Tasks;
+using Jackett.Common.Models;
+using Jackett.Common.Models.IndexerConfig;
+using Jackett.Common.Services.Interfaces;
+using Jackett.Common.Utils;
+using Jackett.Common.Utils.Clients;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using NLog;
+using WebClient = Jackett.Common.Utils.Clients.WebClient;
+
+namespace Jackett.Common.Indexers.Abstract
+{
+    [ExcludeFromCodeCoverage]
+    public abstract class SpeedAppTracker : BaseWebIndexer
+    {
+        private readonly Dictionary<string, string> _apiHeaders = new Dictionary<string, string>
+        {
+            {"Accept", "application/json"},
+            {"Content-Type", "application/json"}
+        };
+        // API DOC: https://speedapp.io/api/doc
+        private string LoginUrl => SiteLink + "api/login";
+        private string SearchUrl => SiteLink + "api/torrent";
+        private string _token;
+
+        private new ConfigurationDataBasicLoginWithEmail configData => (ConfigurationDataBasicLoginWithEmail)base.configData;
+
+        protected SpeedAppTracker(string link, string id, string name, string description,
+            IIndexerConfigurationService configService, WebClient client, Logger logger,
+            IProtectionService p, ICacheService cs, TorznabCapabilities caps)
+            : base(id: id,
+                name: name,
+                description: description,
+                link: link,
+                caps: caps,
+                configService: configService,
+                client: client,
+                logger: logger,
+                p: p,
+                cacheService: cs,
+                configData: new ConfigurationDataBasicLoginWithEmail())
+        {
+        }
+
+        public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
+        {
+            LoadValuesFromJson(configJson);
+
+            await RenewalTokenAsync();
+
+            var releases = await PerformQuery(new TorznabQuery());
+            await ConfigureIfOK(string.Empty, releases.Any(),
+                                () => throw new Exception("Could not find releases."));
+
+            return IndexerConfigurationStatus.Completed;
+        }
+
+        private async Task RenewalTokenAsync()
+        {
+            if (configData.Email.Value == null || configData.Password.Value == null)
+                throw new Exception("Please, check the indexer configuration.");
+            var body = new Dictionary<string, string>
+            {
+                { "username", configData.Email.Value.Trim() },
+                { "password", configData.Password.Value.Trim() }
+            };
+            var jsonData = JsonConvert.SerializeObject(body);
+            var result = await RequestWithCookiesAsync(
+                LoginUrl, method: RequestType.POST, headers: _apiHeaders, rawbody: jsonData);
+            var json = JObject.Parse(result.ContentString);
+            _token = json.Value<string>("token");
+            if (_token == null)
+                throw new Exception(json.Value<string>("message"));
+        }
+
+        protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
+        {
+            var releases = new List<ReleaseInfo>();
+
+            //var categoryMapping = MapTorznabCapsToTrackers(query).Distinct().ToList();
+            var qc = new List<KeyValuePair<string, string>> // NameValueCollection don't support cat[]=19&cat[]=6
+            {
+                {"itemsPerPage", "100"},
+                {"sort", "torrent.createdAt"},
+                {"direction", "desc"}
+            };
+
+            foreach (var cat in MapTorznabCapsToTrackers(query))
+                qc.Add("categories[]", cat);
+
+            if (query.IsImdbQuery)
+                qc.Add("imdbId", query.ImdbID);
+            else
+                qc.Add("search", query.GetQueryString());
+
+            if (string.IsNullOrWhiteSpace(_token)) // fist time login
+                await RenewalTokenAsync();
+
+            var searchUrl = SearchUrl + "?" + qc.GetQueryString();
+            var response = await RequestWithCookiesAsync(searchUrl, headers: GetSearchHeaders());
+            if (response.Status == HttpStatusCode.Unauthorized)
+            {
+                await RenewalTokenAsync(); // re-login
+                response = await RequestWithCookiesAsync(searchUrl, headers: GetSearchHeaders());
+            }
+            else if (response.Status != HttpStatusCode.OK)
+                throw new Exception($"Unknown error in search: {response.ContentString}");
+
+            try
+            {
+                var rows = JArray.Parse(response.ContentString);
+                foreach (var row in rows)
+                {
+                    var id = row.Value<string>("id");
+                    var details = new Uri($"{SiteLink}browse/{id}");
+                    var link = new Uri($"{SiteLink}api/torrent/{id}/download");
+                    var publishDate = DateTime.Parse(row.Value<string>("created_at"), CultureInfo.InvariantCulture);
+                    var cat = row.Value<JToken>("category").Value<string>("id");
+
+                    // "description" field in API has too much HTML code
+                    var description = row.Value<string>("short_description");
+
+                    var posterStr = row.Value<string>("poster");
+                    var poster = Uri.TryCreate(posterStr, UriKind.Absolute, out var posterUri) ? posterUri : null;
+
+                    var dlVolumeFactor = row.Value<double>("download_volume_factor");
+                    var ulVolumeFactor = row.Value<double>("upload_volume_factor");
+
+                    var release = new ReleaseInfo
+                    {
+                        Title = row.Value<string>("name"),
+                        Link = link,
+                        Details = details,
+                        Guid = details,
+                        Category =  MapTrackerCatToNewznab(cat),
+                        PublishDate = publishDate,
+                        Description = description,
+                        Poster = poster,
+                        Size = row.Value<long>("size"),
+                        Grabs = row.Value<long>("times_completed"),
+                        Seeders = row.Value<int>("seeders"),
+                        Peers = row.Value<int>("leechers") + row.Value<int>("seeders"),
+                        DownloadVolumeFactor = dlVolumeFactor,
+                        UploadVolumeFactor = ulVolumeFactor,
+                        MinimumRatio = 1,
+                        MinimumSeedTime = 172800 // 48 hours
+                    };
+
+                    releases.Add(release);
+                }
+            }
+            catch (Exception ex)
+            {
+                OnParseError(response.ContentString, ex);
+            }
+            return releases;
+        }
+
+        public override async Task<byte[]> Download(Uri link)
+        {
+            var response = await RequestWithCookiesAsync(link.ToString(), headers: GetSearchHeaders());
+            if (response.Status == HttpStatusCode.Unauthorized)
+            {
+                await RenewalTokenAsync();
+                response = await RequestWithCookiesAsync(link.ToString(), headers: GetSearchHeaders());
+            }
+            else if (response.Status != HttpStatusCode.OK)
+                throw new Exception($"Unknown error in download: {response.ContentBytes}");
+            return response.ContentBytes;
+        }
+
+        private Dictionary<string, string> GetSearchHeaders() => new Dictionary<string, string>
+        {
+            {"Authorization", $"Bearer {_token}"}
+        };
+    }
+}

src/Jackett.Common/Indexers/RetroFlix.cs

@@ -0,0 +1,66 @@
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Text;
+using Jackett.Common.Indexers.Abstract;
+using Jackett.Common.Models;
+using Jackett.Common.Services.Interfaces;
+using NLog;
+using WebClient = Jackett.Common.Utils.Clients.WebClient;
+
+namespace Jackett.Common.Indexers
+{
+    [ExcludeFromCodeCoverage]
+    public class RetroFlix : SpeedAppTracker
+    {
+        public override string[] LegacySiteLinks { get; protected set; } = {
+            "https://retroflix.net/"
+        };
+
+        public RetroFlix(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps,
+            ICacheService cs)
+            : base(
+                id: "retroflix",
+                name: "RetroFlix",
+                description: "Private Torrent Tracker for Classic Movies / TV / General Releases",
+                link: "https://retroflix.club/",
+                caps: new TorznabCapabilities
+                {
+                    TvSearchParams = new List<TvSearchParam>
+                    {
+                        TvSearchParam.Q, TvSearchParam.Season, TvSearchParam.Ep, TvSearchParam.ImdbId
+                    },
+                    MovieSearchParams = new List<MovieSearchParam>
+                    {
+                        MovieSearchParam.Q, MovieSearchParam.ImdbId
+                    },
+                    MusicSearchParams = new List<MusicSearchParam>
+                    {
+                        MusicSearchParam.Q
+                    },
+                    BookSearchParams = new List<BookSearchParam>
+                    {
+                        BookSearchParam.Q
+                    }
+                },
+                configService: configService,
+                client: wc,
+                logger: l,
+                p: ps,
+                cs: cs)
+        {
+            Encoding = Encoding.UTF8;
+            Language = "en-us";
+            Type = "private";
+
+            // requestDelay for API Limit (1 request per 2 seconds)
+            webclient.requestDelay = 2.1;
+ 
+            AddCategoryMapping(401, TorznabCatType.Movies, "Movies");
+            AddCategoryMapping(402, TorznabCatType.TV, "TV Series");
+            AddCategoryMapping(406, TorznabCatType.AudioVideo, "Music Videos");
+            AddCategoryMapping(407, TorznabCatType.TVSport, "Sports");
+            AddCategoryMapping(409, TorznabCatType.Books, "Books");
+            AddCategoryMapping(408, TorznabCatType.Audio, "HQ Audio");
+        }
+    }
+}

src/Jackett.Common/Indexers/SpeedApp.cs

@@ -1,38 +1,17 @@
-using System;
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
-using System.Globalization;
-using System.Linq;
-using System.Net;
 using System.Text;
-using System.Threading.Tasks;
+using Jackett.Common.Indexers.Abstract;
 using Jackett.Common.Models;
-using Jackett.Common.Models.IndexerConfig;
 using Jackett.Common.Services.Interfaces;
-using Jackett.Common.Utils;
-using Jackett.Common.Utils.Clients;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Linq;
 using NLog;
 using WebClient = Jackett.Common.Utils.Clients.WebClient;
 
 namespace Jackett.Common.Indexers
 {
     [ExcludeFromCodeCoverage]
-    public class SpeedApp : BaseWebIndexer
+    public class SpeedApp : SpeedAppTracker
     {
-        private readonly Dictionary<string, string> _apiHeaders = new Dictionary<string, string>
-        {
-            {"Accept", "application/json"},
-            {"Content-Type", "application/json"}
-        };
-        // API DOC: https://speedapp.io/api/doc
-        private string LoginUrl => SiteLink + "api/login";
-        private string SearchUrl => SiteLink + "api/torrent";
-        private string _token;
-
-        private new ConfigurationDataBasicLoginWithEmail configData => (ConfigurationDataBasicLoginWithEmail)base.configData;
-
         public override string[] LegacySiteLinks { get; protected set; } = {
             "https://www.icetorrent.org/",
             "https://icetorrent.org/",
@@ -74,8 +53,7 @@ namespace Jackett.Common.Indexers
                 client: wc,
                 logger: l,
                 p: ps,
-                cacheService: cs,
+                cs: cs)
-                configData: new ConfigurationDataBasicLoginWithEmail())
         {
             Encoding = Encoding.UTF8;
             Language = "ro-ro";
@@ -128,135 +106,5 @@ namespace Jackett.Common.Indexers
             AddCategoryMapping(50, TorznabCatType.XXX, "XXX Packs");
             AddCategoryMapping(51, TorznabCatType.XXX, "XXX SD");
         }
-
-        public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
-        {
-            LoadValuesFromJson(configJson);
-
-            await RenewalTokenAsync();
-
-            var releases = await PerformQuery(new TorznabQuery());
-            await ConfigureIfOK(string.Empty, releases.Any(),
-                                () => throw new Exception("Could not find releases."));
-
-            return IndexerConfigurationStatus.Completed;
-        }
-
-        private async Task RenewalTokenAsync()
-        {
-            var body = new Dictionary<string, string>
-            {
-                { "username", configData.Email.Value.Trim() },
-                { "password", configData.Password.Value.Trim() }
-            };
-            var jsonData = JsonConvert.SerializeObject(body);
-            var result = await RequestWithCookiesAsync(
-                LoginUrl, method: RequestType.POST, headers: _apiHeaders, rawbody: jsonData);
-            var json = JObject.Parse(result.ContentString);
-            _token = json.Value<string>("token");
-            if (_token == null)
-                throw new Exception(json.Value<string>("message"));
-        }
-
-        protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
-        {
-            var releases = new List<ReleaseInfo>();
-
-            //var categoryMapping = MapTorznabCapsToTrackers(query).Distinct().ToList();
-            var qc = new List<KeyValuePair<string, string>> // NameValueCollection don't support cat[]=19&cat[]=6
-            {
-                {"itemsPerPage", "100"},
-                {"sort", "torrent.createdAt"},
-                {"direction", "desc"}
-            };
-
-            foreach (var cat in MapTorznabCapsToTrackers(query))
-                qc.Add("categories[]", cat);
-
-            if (query.IsImdbQuery)
-                qc.Add("imdbId", query.ImdbID);
-            else
-                qc.Add("search", query.GetQueryString());
-
-            if (string.IsNullOrWhiteSpace(_token)) // fist time login
-                await RenewalTokenAsync();
-
-            var searchUrl = SearchUrl + "?" + qc.GetQueryString();
-            var response = await RequestWithCookiesAsync(searchUrl, headers: GetSearchHeaders());
-            if (response.Status == HttpStatusCode.Unauthorized)
-            {
-                await RenewalTokenAsync(); // re-login
-                response = await RequestWithCookiesAsync(searchUrl, headers: GetSearchHeaders());
-            }
-            else if (response.Status != HttpStatusCode.OK)
-                throw new Exception($"Unknown error in search: {response.ContentString}");
-
-            try
-            {
-                var rows = JArray.Parse(response.ContentString);
-                foreach (var row in rows)
-                {
-                    var id = row.Value<string>("id");
-                    var details = new Uri($"{SiteLink}browse/{id}");
-                    var link = new Uri($"{SiteLink}api/torrent/{id}/download");
-                    var publishDate = DateTime.Parse(row.Value<string>("created_at"), CultureInfo.InvariantCulture);
-                    var cat = row.Value<JToken>("category").Value<string>("id");
-
-                    // "description" field in API has too much HTML code
-                    var description = row.Value<string>("short_description");
-
-                    var posterStr = row.Value<string>("poster");
-                    var poster = Uri.TryCreate(posterStr, UriKind.Absolute, out var posterUri) ? posterUri : null;
-
-                    var dlVolumeFactor = row.Value<double>("download_volume_factor");
-                    var ulVolumeFactor = row.Value<double>("upload_volume_factor");
-
-                    var release = new ReleaseInfo
-                    {
-                        Title = row.Value<string>("name"),
-                        Link = link,
-                        Details = details,
-                        Guid = details,
-                        Category =  MapTrackerCatToNewznab(cat),
-                        PublishDate = publishDate,
-                        Description = description,
-                        Poster = poster,
-                        Size = row.Value<long>("size"),
-                        Grabs = row.Value<long>("times_completed"),
-                        Seeders = row.Value<int>("seeders"),
-                        Peers = row.Value<int>("leechers") + row.Value<int>("seeders"),
-                        DownloadVolumeFactor = dlVolumeFactor,
-                        UploadVolumeFactor = ulVolumeFactor,
-                        MinimumRatio = 1,
-                        MinimumSeedTime = 172800 // 48 hours
-                    };
-
-                    releases.Add(release);
-                }
-            }
-            catch (Exception ex)
-            {
-                OnParseError(response.ContentString, ex);
-            }
-            return releases;
-        }
-
-        public override async Task<byte[]> Download(Uri link)
-        {
-            var response = await RequestWithCookiesAsync(link.ToString(), headers: GetSearchHeaders());
-            if (response.Status == HttpStatusCode.Unauthorized)
-            {
-                await RenewalTokenAsync();
-                response = await RequestWithCookiesAsync(link.ToString(), headers: GetSearchHeaders());
-            }
-            else if (response.Status != HttpStatusCode.OK)
-                throw new Exception($"Unknown error in download: {response.ContentBytes}");
-            return response.ContentBytes;
-        }
-
-        private Dictionary<string, string> GetSearchHeaders() => new Dictionary<string, string>
-        {
-            {"Authorization", $"Bearer {_token}"}
-        };
     }
 }

src/Jackett.Updater/Program.cs

@@ -368,6 +368,7 @@ namespace Jackett.Updater
                 "Definitions/rapidetracker.yml",
                 "Definitions/rarbg.yml", // migrated to C#
                 "Definitions/redtopia.yml",
+                "Definitions/retroflix.yml", // migrated to C#
                 "Definitions/rgu.yml",
                 "Definitions/rns.yml", // site merged with audiobooktorrents
                 "Definitions/rockethd.yml",