FileList: Add API functionality. Resolves #7004 Resolves #5190 (#7987)

src/Jackett.Common/Indexers/FileList.cs

@@ -1,12 +1,8 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.Specialized;
-using System.Globalization;
-using System.Linq;
 using System.Text;
-using System.Text.RegularExpressions;
 using System.Threading.Tasks;
-using AngleSharp.Html.Parser;
 using Jackett.Common.Models;
 using Jackett.Common.Models.IndexerConfig.Bespoke;
 using Jackett.Common.Services.Interfaces;
@@ -19,27 +15,27 @@ namespace Jackett.Common.Indexers
 {
     public class FileList : BaseWebIndexer
     {
-        public override string[] LegacySiteLinks { get; protected set; } = {
+        public override string[] LegacySiteLinks { get; protected set; } =
+        {
             "http://filelist.ro/",
             "https://filelist.ro/",
             "https://flro.org/",
             "http://flro.org/",
         };
 
-        private string LoginUrl => SiteLink + "takelogin.php";
+        private string ApiUrl => SiteLink + "api.php";
-        private string BrowseUrl => SiteLink + "browse.php";
+        private string DetailsUrl => SiteLink + "details.php";
 
-        private new ConfigurationDataFileList configData
+        private new ConfigurationDataFileList configData => (ConfigurationDataFileList)base.configData;
-        {
-            get => (ConfigurationDataFileList)base.configData;
-            set => base.configData = value;
-        }
 
         public FileList(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps)
-            : base(name: "FileList",
+            : base("FileList",
                 description: "The best Romanian site.",
                 link: "https://filelist.io/",
-                caps: TorznabUtil.CreateDefaultTorznabTVCaps(),
+                caps: new TorznabCapabilities
+                {
+                    SupportsImdbMovieSearch = true
+                },
                 configService: configService,
                 client: wc,
                 logger: l,
@@ -50,161 +46,152 @@ namespace Jackett.Common.Indexers
             Language = "ro-ro";
             Type = "private";
 
-            TorznabCaps.SupportsImdbMovieSearch = true;
+            AddCategoryMapping(1, TorznabCatType.MoviesSD, "Filme SD");
-
-            AddCategoryMapping(24, TorznabCatType.TVAnime, "Anime");
-            AddCategoryMapping(11, TorznabCatType.Audio, "Audio");
-            AddCategoryMapping(15, TorznabCatType.TV, "Desene");
-            AddCategoryMapping(18, TorznabCatType.Other, "Diverse");
-            AddCategoryMapping(16, TorznabCatType.Books, "Docs");
-            AddCategoryMapping(25, TorznabCatType.Movies3D, "Filme 3D");
-            AddCategoryMapping(6, TorznabCatType.MoviesHD, "Filme 4K");
-            AddCategoryMapping(26, TorznabCatType.MoviesBluRay, "Filme 4K Blu-Ray");
-            AddCategoryMapping(20, TorznabCatType.MoviesBluRay, "Filme Blu-Ray");
             AddCategoryMapping(2, TorznabCatType.MoviesDVD, "Filme DVD");
             AddCategoryMapping(3, TorznabCatType.MoviesForeign, "Filme DVD-RO");
             AddCategoryMapping(4, TorznabCatType.MoviesHD, "Filme HD");
-            AddCategoryMapping(19, TorznabCatType.MoviesForeign, "Filme HD-RO");
-            AddCategoryMapping(1, TorznabCatType.MoviesSD, "Filme SD");
             AddCategoryMapping(5, TorznabCatType.AudioLossless, "FLAC");
-            AddCategoryMapping(10, TorznabCatType.Console, "Jocuri Console");
+            AddCategoryMapping(6, TorznabCatType.MoviesUHD, "Filme 4K");
-            AddCategoryMapping(9, TorznabCatType.PCGames, "Jocuri PC");
-            AddCategoryMapping(17, TorznabCatType.PC, "Linux");
-            AddCategoryMapping(22, TorznabCatType.PCPhoneOther, "Mobile");
-            AddCategoryMapping(8, TorznabCatType.PC, "Programe");
-            AddCategoryMapping(27, TorznabCatType.TVHD, "Seriale 4K");
-            AddCategoryMapping(21, TorznabCatType.TVHD, "Seriale HD");
-            AddCategoryMapping(23, TorznabCatType.TVSD, "Seriale SD");
-            AddCategoryMapping(13, TorznabCatType.TVSport, "Sport");
-            AddCategoryMapping(12, TorznabCatType.AudioVideo, "Videoclip");
             AddCategoryMapping(7, TorznabCatType.XXX, "XXX");
+            AddCategoryMapping(8, TorznabCatType.PC, "Programe");
+            AddCategoryMapping(9, TorznabCatType.PCGames, "Jocuri PC");
+            AddCategoryMapping(10, TorznabCatType.Console, "Jocuri Console");
+            AddCategoryMapping(11, TorznabCatType.Audio, "Audio");
+            AddCategoryMapping(12, TorznabCatType.AudioVideo, "Videoclip");
+            AddCategoryMapping(13, TorznabCatType.TVSport, "Sport");
+            AddCategoryMapping(13, TorznabCatType.TVSport, "Sport");
+            AddCategoryMapping(15, TorznabCatType.TV, "Desene");
+            AddCategoryMapping(16, TorznabCatType.Books, "Docs");
+            AddCategoryMapping(17, TorznabCatType.PC, "Linux");
+            AddCategoryMapping(18, TorznabCatType.Other, "Diverse");
+            AddCategoryMapping(19, TorznabCatType.MoviesForeign, "Filme HD-RO");
+            AddCategoryMapping(20, TorznabCatType.MoviesBluRay, "Filme Blu-Ray");
+            AddCategoryMapping(21, TorznabCatType.TVHD, "Seriale HD");
+            AddCategoryMapping(22, TorznabCatType.PCPhoneOther, "Mobile");
+            AddCategoryMapping(23, TorznabCatType.TVSD, "Seriale SD");
+            AddCategoryMapping(24, TorznabCatType.TVAnime, "Anime");
+            AddCategoryMapping(25, TorznabCatType.Movies3D, "Filme 3D");
+            AddCategoryMapping(26, TorznabCatType.MoviesBluRay, "Filme 4K Blu-Ray");
+            AddCategoryMapping(27, TorznabCatType.TVUHD, "Seriale 4K");
         }
 
         public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
         {
             LoadValuesFromJson(configJson);
-            var responseFirstPage = await RequestStringWithCookiesAndRetry(SiteLink + "login.php?returnto=%2F", "");
+            logger.Info("Testing provider filelist...");
-            var parser = new HtmlParser();
+            var pingResponse = await CallProviderAsync(new TorznabQuery());
-            var domFirstPage = parser.ParseDocument(responseFirstPage.Content);
-            var validator = domFirstPage.QuerySelector("input[name =\"validator\"]").GetAttribute("value");
-            var pairs = new Dictionary<string, string> {
-                { "validator", validator},
-                { "username", configData.Username.Value },
-                { "password", configData.Password.Value }
-            };
 
-            var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, responseFirstPage.Cookies, true, null, LoginUrl);
+            try
-            await ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("logout.php"), () =>
             {
-                var dom = parser.ParseDocument(result.Content);
+                var json = JArray.Parse(pingResponse);
-                var errorMessage = dom.QuerySelector(".main").TextContent.Trim();
+                if (json.Count > 0)
-                throw new ExceptionWithConfigData(errorMessage, configData);
+                {
-            });
+                    IsConfigured = true;
+                    SaveConfig();
+                    return IndexerConfigurationStatus.Completed;
+                }
+            }
+            catch (Exception ex)
+            {
+                throw new ExceptionWithConfigData(ex.Message, configData);
+            }
+
             return IndexerConfigurationStatus.RequiresTesting;
         }
 
         protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
         {
             var releases = new List<ReleaseInfo>();
-            var searchUrl = BrowseUrl;
+            var response = await CallProviderAsync(query);
-            var searchString = query.GetQueryString();
-            var cat = MapTorznabCapsToTrackers(query).FirstIfSingleOrDefault("0");
 
-            var queryCollection = new NameValueCollection();
+            if (response.StartsWith("{\"error\""))
+                throw new ExceptionWithConfigData(response, configData);
 
-            if (query.ImdbID != null)
-                queryCollection.Add("search", query.ImdbID);
-            else if (!string.IsNullOrWhiteSpace(searchString))
-                queryCollection.Add("search", searchString);
-
-            queryCollection.Add("cat", cat);
-            queryCollection.Add("searchin", "1");
-            queryCollection.Add("sort", "2");
-
-            searchUrl += "?" + queryCollection.GetQueryString();
-
-            var response = await RequestStringWithCookiesAndRetry(searchUrl, null, BrowseUrl);
-
-            // Occasionally the cookies become invalid, login again if that happens
-            if (response.IsRedirect)
-            {
-                await ApplyConfiguration(null);
-                response = await RequestStringWithCookiesAndRetry(searchUrl, null, BrowseUrl);
-            }
-
-            var results = response.Content;
             try
             {
-                var parser = new HtmlParser();
+                var json = JArray.Parse(response);
-                var dom = parser.ParseDocument(results);
+                foreach (var row in json)
-                var globalFreeLeech = dom.QuerySelectorAll("div.globalFreeLeech").Any();
-                var rows = dom.QuerySelectorAll(".torrentrow");
-                foreach (var row in rows)
                 {
-                    var release = new ReleaseInfo();
+                    var detailsUri = new Uri(DetailsUrl + "?id=" + (string)row["id"]);
-
+                    var seeders = (int)row["seeders"];
-                    var qTitleLink = row.QuerySelector(".torrenttable:nth-of-type(2) a");
+                    var peers = seeders + (int)row["leechers"];
-                    release.Title = row.QuerySelector(".torrenttable:nth-of-type(2) b").TextContent;
+                    var publishDate = DateTimeUtil.FromFuzzyTime((string)row["upload_date"] + " +0200");
-                    var longtitle = row.QuerySelector(".torrenttable:nth-of-type(2) a[title]").GetAttribute("title");
+                    var downloadVolumeFactor = (int)row["freeleech"] == 1 ? 0 : 1;
-                    if (!string.IsNullOrEmpty(longtitle) && !longtitle.Contains("<")) // releases with cover image have no full title
+                    var imdbId = ((JObject)row).ContainsKey("imdb") ? ParseUtil.GetImdbID((string)row["imdb"]) : null;
-                        release.Title = longtitle;
+                    var link = new Uri((string)row["download_link"]);
-
+                    var release = new ReleaseInfo
-                    if (query.ImdbID == null && !query.MatchQueryStringAND(release.Title))
-                        continue;
-
-                    release.Description = row.QuerySelector(".torrenttable:nth-of-type(2) > span > font.small")?.TextContent;
-
-                    var tooltip = qTitleLink.GetAttribute("title");
-                    if (!string.IsNullOrEmpty(tooltip))
                     {
-                        var imgRegexp = new Regex("src='(.*?)'");
+                        Title = (string)row["name"],
-                        var imgRegexpMatch = imgRegexp.Match(tooltip);
+                        Comments = detailsUri,
-                        if (imgRegexpMatch.Success)
+                        Link = link,
-                            release.BannerUrl = new Uri(imgRegexpMatch.Groups[1].Value);
+                        Category = MapTrackerCatDescToNewznab((string)row["category"]),
-                    }
+                        Size = (long)row["size"],
-
+                        Files = (long)row["files"],
-                    release.Guid = new Uri(SiteLink + qTitleLink.GetAttribute("href"));
+                        Grabs = (long)row["times_completed"],
-                    release.Comments = release.Guid;
+                        Seeders = seeders,
-
+                        Peers = peers,
-                    //22:05:3716/02/2013
+                        MinimumRatio = 1,
-                    var dateStr = row.QuerySelector(".torrenttable:nth-of-type(6)").TextContent.Trim() + " +0200";
+                        MinimumSeedTime = 172800, //48 hours
-                    release.PublishDate = DateTime.ParseExact(dateStr, "H:mm:ssdd/MM/yyyy zzz", CultureInfo.InvariantCulture);
+                        PublishDate = publishDate,
-
+                        UploadVolumeFactor = 1,
-                    var qLink = row.QuerySelector("a[href^=\"download.php?id=\"]");
+                        DownloadVolumeFactor = downloadVolumeFactor,
-                    release.Link = new Uri(SiteLink + qLink.GetAttribute("href").Replace("&usetoken=1", ""));
+                        Guid = detailsUri,
-
+                        Imdb = imdbId
-                    var sizeStr = row.QuerySelector(".torrenttable:nth-of-type(7)").TextContent.Trim();
+                    };
-                    release.Size = ReleaseInfo.GetBytes(sizeStr);
-
-                    var grabs = row.QuerySelector(".torrenttable:nth-of-type(8)").TextContent.Replace("times", "").Trim();
-                    release.Grabs = ParseUtil.CoerceLong(grabs);
-
-                    release.Seeders = ParseUtil.CoerceInt(row.QuerySelector(".torrenttable:nth-of-type(9)").TextContent.Trim());
-                    release.Peers = ParseUtil.CoerceInt(row.QuerySelector(".torrenttable:nth-of-type(10)").TextContent.Trim()) + release.Seeders;
-
-                    var catId = row.QuerySelector(".torrenttable:nth-of-type(1) a").GetAttribute("href").Substring(15);
-                    release.Category = MapTrackerCatToNewznab(catId);
-
-                    if (globalFreeLeech || row.QuerySelectorAll("img[alt=\"FreeLeech\"]").Any())
-                        release.DownloadVolumeFactor = 0;
-                    else
-                        release.DownloadVolumeFactor = 1;
-
-                    release.UploadVolumeFactor = 1;
-
-                    // Skip Romanian releases
-                    if (release.Category.Contains(TorznabCatType.MoviesForeign.ID) && !configData.IncludeRomanianReleases.Value)
-                        continue;
-
                     releases.Add(release);
                 }
-            }
-            catch (Exception ex)
-            {
-                OnParseError(results, ex);
-            }
 
                 return releases;
             }
+            catch (Exception ex)
+            {
+                OnParseError(response, ex);
+            }
+
+            return releases;
+        }
+
+        private async Task<string> CallProviderAsync(TorznabQuery query)
+        {
+            var searchUrl = ApiUrl;
+            var searchString = query.GetQueryString();
+            var cat = string.Join(",", MapTorznabCapsToTrackers(query));
+            var queryCollection = new NameValueCollection
+            {
+                {"category", cat}
+            };
+
+            if (query.IsImdbQuery)
+            {
+                queryCollection.Add("type", "imdb");
+                queryCollection.Add("query", query.ImdbID);
+                queryCollection.Add("action", "search-torrents");
+            }
+            else if (!string.IsNullOrWhiteSpace(searchString))
+            {
+                queryCollection.Add("type", "name");
+                queryCollection.Add("query", searchString);
+                queryCollection.Add("action", "search-torrents");
+            }
+            else
+                queryCollection.Add("action", "latest-torrents");
+
+            searchUrl += "?" + queryCollection.GetQueryString();
+            try
+            {
+                var auth = Convert.ToBase64String(Encoding.UTF8.GetBytes(configData.Username.Value + ":" + configData.Passkey.Value));
+                var headers = new Dictionary<string, string>
+                {
+                    {"Authorization", "Basic " + auth}
+                };
+                var response = await RequestStringWithCookies(searchUrl, headers: headers);
+                return response.Content;
+            }
+            catch (Exception inner)
+            {
+                throw new Exception("Error calling provider filelist", inner);
             }
         }
+    }
+}
+

src/Jackett.Common/Models/IndexerConfig/Bespoke/ConfigurationDataFileList.cs

@@ -1,14 +1,14 @@
 namespace Jackett.Common.Models.IndexerConfig.Bespoke
 {
-    internal class ConfigurationDataFileList : ConfigurationDataBasicLogin
+    internal class ConfigurationDataFileList : ConfigurationDataUserPasskey
     {
         public BoolItem IncludeRomanianReleases { get; private set; }
         public DisplayItem CatWarning { get; private set; }
 
         public ConfigurationDataFileList()
-            : base()
+            : base("Go into your filelist profile and copy the passkey.")
         {
-            IncludeRomanianReleases = new BoolItem() { Name = "IncludeRomanianReleases", Value = false };
+            IncludeRomanianReleases = new BoolItem {Name = "IncludeRomanianReleases", Value = false};
             CatWarning = new DisplayItem("When mapping TV ensure you add category 5000 in addition to 5030, 5040.") {Name = "CatWarning"};
         }
     }