fix(indexers): rewritted scenefz from scratch (#1684)

src/Jackett/Indexers/SceneFZ.cs

@@ -1,186 +1,801 @@
-using Jackett.Utils.Clients;
+using System;
-using NLog;
+using System.Collections.Generic;
+using System.Collections.Specialized;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+using System.Web;
+using CsQuery;
+using Jackett.Models;
+using Jackett.Models.IndexerConfig.Bespoke;
 using Jackett.Services;
 using Jackett.Utils;
-using Jackett.Models;
+using Jackett.Utils.Clients;
-using System.Threading.Tasks;
+using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
-using System.Collections.Generic;
+using NLog;
-using CsQuery;
-using System.Web;
-using System;
-using System.Text;
-using System.Globalization;
-using Jackett.Models.IndexerConfig;
 
 namespace Jackett.Indexers
 {
-    public class SceneFZ : BaseWebIndexer
+    /// <summary>
+    /// Provider for SceneFZ
+    /// </summary>
+    public class SceneFZ : BaseCachingWebIndexer
     {
-        string LoginUrl { get { return SiteLink + "takelogin.php"; } }
+        private string LoginUrl { get { return SiteLink + "login.php"; } }
+        private string LoginCheckUrl { get { return SiteLink + "takelogin.php"; } }
+        private string SearchUrl { get { return SiteLink + "torrents.php"; } }
+        private bool Latency { get { return ConfigData.Latency.Value; } }
+        private bool DevMode { get { return ConfigData.DevMode.Value; } }
+        private bool CacheMode { get { return ConfigData.HardDriveCache.Value; } }
+        private static string Directory => Path.Combine(Path.GetTempPath(), Assembly.GetExecutingAssembly().GetName().Name.ToLower(), MethodBase.GetCurrentMethod().DeclaringType?.Name.ToLower());
 
-        string BrowseUrl { get { return SiteLink + "ajax_browse.php"; } }
+        private Dictionary<string, string> emulatedBrowserHeaders = new Dictionary<string, string>();
+        private CQ fDom = null;
 
-        new ConfigurationDataBasicLogin configData
+        private ConfigurationDataSceneFZ ConfigData
         {
-            get { return (ConfigurationDataBasicLogin)base.configData; }
+            get { return (ConfigurationDataSceneFZ)configData; }
             set { base.configData = value; }
         }
 
-        public SceneFZ(IIndexerConfigurationService configService, IWebClient wc, Logger l, IProtectionService ps)
+        public SceneFZ(IIndexerConfigurationService configService, IWebClient w, Logger l, IProtectionService ps)
-            : base(name: "SceneFZ",
+            : base(
-                   description: "Torrent tracker. Tracking over 50.000 torrent files.",
+                name: "SceneFZ",
-                   link: "http://scenefz.me/",
+                description: "Torrent tracker. Tracking over 50.000 torrent files.",
-                   caps: TorznabUtil.CreateDefaultTorznabTVCaps(),
+                link: "https://scenefz.me/",
-                   configService: configService,
+                caps: TorznabUtil.CreateDefaultTorznabTVCaps(),
-                   client: wc,
+                configService: configService,
-                   logger: l,
+                client: w,
-                   p: ps,
+                logger: l,
-                   configData: new ConfigurationDataBasicLogin())
+                p: ps,
+                configData: new ConfigurationDataSceneFZ())
         {
             Encoding = Encoding.UTF8;
             Language = "ro-ro";
             Type = "private";
 
-            this.configData.Instructions.Value = "The published date is only available if you set \"Torrent Listing\" to Complex is your profile.";
+            // Clean capabilities
+            TorznabCaps.Categories.Clear();
 
-            AddCategoryMapping("mc32", TorznabCatType.Movies); // Movies
+            AddCategoryMapping(1, TorznabCatType.TVAnime);
-            AddCategoryMapping("scat22", TorznabCatType.MoviesBluRay); // BluRay
+            AddCategoryMapping(49, TorznabCatType.Other);
-            AddCategoryMapping("scat40", TorznabCatType.MoviesSD); // Xvid
+            AddCategoryMapping(43, TorznabCatType.TVDocumentary);
-            AddCategoryMapping("scat41", TorznabCatType.MoviesDVD); // Dvd
+            AddCategoryMapping(35, TorznabCatType.Console);
-            AddCategoryMapping("scat47", TorznabCatType.MoviesHD); // HD
+            AddCategoryMapping(30, TorznabCatType.PC);
-            AddCategoryMapping("scat52", TorznabCatType.MoviesDVD); // DVD-RO
+            AddCategoryMapping(13, TorznabCatType.TVHD);
-            AddCategoryMapping("scat53", TorznabCatType.MoviesHD); // HD-RO
+            AddCategoryMapping(14, TorznabCatType.TVHD);
-            AddCategoryMapping("scat54", TorznabCatType.MoviesBluRay); // BluRay-RO
+            AddCategoryMapping(44, TorznabCatType.Other);
-            AddCategoryMapping("scat55", TorznabCatType.MoviesSD); // XVID-RO
+            AddCategoryMapping(45, TorznabCatType.PC);
-            AddCategoryMapping("scat60", TorznabCatType.MoviesOther); // Sport
+            AddCategoryMapping(15, TorznabCatType.PCPhoneOther);
-            AddCategoryMapping("mc33", TorznabCatType.TV); // TV
+            AddCategoryMapping(61, TorznabCatType.Movies3D);
-            AddCategoryMapping("scat66", TorznabCatType.TVSD); // SD
+            AddCategoryMapping(62, TorznabCatType.Movies3D);
-            AddCategoryMapping("scat67", TorznabCatType.TVSD); // SD-RO
+            AddCategoryMapping(5, TorznabCatType.MoviesBluRay);
-            AddCategoryMapping("scat68", TorznabCatType.TVHD); // HD
+            AddCategoryMapping(6, TorznabCatType.MoviesBluRay);
-            AddCategoryMapping("scat69", TorznabCatType.TVHD); // HDTV-RO
+            AddCategoryMapping(9, TorznabCatType.MoviesDVD);
-            AddCategoryMapping("mc30", TorznabCatType.Console); // Games
+            AddCategoryMapping(10, TorznabCatType.MoviesDVD);
-            AddCategoryMapping("scat58", TorznabCatType.ConsolePS3); // PS2
+            AddCategoryMapping(11, TorznabCatType.MoviesHD);
-            AddCategoryMapping("scat16", TorznabCatType.PCGames); // Pc-Iso
+            AddCategoryMapping(12, TorznabCatType.MoviesHD);
-            AddCategoryMapping("scat17", TorznabCatType.Console); // Misc
+            AddCategoryMapping(24, TorznabCatType.MoviesSD);
-            AddCategoryMapping("scat18", TorznabCatType.PCGames); // Pc-Rip
+            AddCategoryMapping(25, TorznabCatType.MoviesSD);
-            AddCategoryMapping("scat19", TorznabCatType.Console); // Consoles
+            AddCategoryMapping(27, TorznabCatType.XXX);
-            AddCategoryMapping("scat57", TorznabCatType.ConsoleXbox360); // Xbox 360
+            AddCategoryMapping(28, TorznabCatType.Audio);
-            AddCategoryMapping("scat46", TorznabCatType.Console); // Oldies
+            AddCategoryMapping(64, TorznabCatType.AudioLossless);
-            AddCategoryMapping("scat59", TorznabCatType.ConsolePS3); // PS3
+            AddCategoryMapping(29, TorznabCatType.AudioVideo);
-            AddCategoryMapping("mc31", TorznabCatType.PC); // Soft
+            AddCategoryMapping(26, TorznabCatType.PCISO);
-            AddCategoryMapping("scat20", TorznabCatType.PC); // Pc-Iso
+            AddCategoryMapping(22, TorznabCatType.TVSport);
-            AddCategoryMapping("scat21", TorznabCatType.PC); // Misc
+            AddCategoryMapping(20, TorznabCatType.TVSD);
-            AddCategoryMapping("scat48", TorznabCatType.PCMac); // Mac OS
+            AddCategoryMapping(21, TorznabCatType.TVSD);
-            AddCategoryMapping("mc27", TorznabCatType.Audio); // Music
-            AddCategoryMapping("scat8", TorznabCatType.AudioMP3); // MP3
-            AddCategoryMapping("scat45", TorznabCatType.AudioVideo); // Videoclips
-            AddCategoryMapping("scat61", TorznabCatType.AudioLossless); // FLAC
-            AddCategoryMapping("mc35", TorznabCatType.PCPhoneOther); // Mobile
-            AddCategoryMapping("scat44", TorznabCatType.PCPhoneOther); // Misc
-            AddCategoryMapping("scat64", TorznabCatType.PCPhoneIOS); // iOS
-            AddCategoryMapping("scat65", TorznabCatType.PCPhoneAndroid); // Android
-            AddCategoryMapping("mc28", TorznabCatType.TVAnime); // Anime
-            AddCategoryMapping("scat13", TorznabCatType.TVAnime); // Tv-Eps
-            AddCategoryMapping("scat12", TorznabCatType.TVAnime); // Cartoons
-            AddCategoryMapping("mc29", TorznabCatType.TVDocumentary); // Docs
-            AddCategoryMapping("scat14", TorznabCatType.Books); // Books
-            AddCategoryMapping("scat15", TorznabCatType.Other); // Misc
-            AddCategoryMapping("mc36", TorznabCatType.PC0day); // 0Day
-            AddCategoryMapping("mc34", TorznabCatType.XXX); // XXX 18
-            AddCategoryMapping("scat33", TorznabCatType.Other); // Images
-            AddCategoryMapping("scat34", TorznabCatType.Other); // Video
         }
 
+        /// <summary>
+        /// Configure our SceneFZ Provider
+        /// </summary>
+        /// <param name="configJson">Our params in Json</param>
+        /// <returns>Configuration state</returns>
         public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
         {
-            LoadValuesFromJson(configJson);
+            try
-            var pairs = new Dictionary<string, string>
             {
-                { "username", configData.Username.Value },
+                // Retrieve config values set by Jackett's user
-                { "password", configData.Password.Value }
+                LoadValuesFromJson(configJson);
+
+                // Check & Validate Config
+                validateConfig();
+
+                // Setting our data for a better emulated browser (maximum security)
+                // TODO: Encoded Content not supported by Jackett at this time
+                // emulatedBrowserHeaders.Add("Accept-Encoding", "gzip, deflate");
+
+                // If we want to simulate a browser
+                if (ConfigData.Browser.Value)
+                {
+
+                    // Clean headers
+                    emulatedBrowserHeaders.Clear();
+
+                    // Inject headers
+                    emulatedBrowserHeaders.Add("Accept", ConfigData.HeaderAccept.Value);
+                    emulatedBrowserHeaders.Add("Accept-Language", ConfigData.HeaderAcceptLang.Value);
+                    emulatedBrowserHeaders.Add("DNT", Convert.ToInt32(ConfigData.HeaderDNT.Value).ToString());
+                    emulatedBrowserHeaders.Add("Upgrade-Insecure-Requests", Convert.ToInt32(ConfigData.HeaderUpgradeInsecure.Value).ToString());
+                    emulatedBrowserHeaders.Add("User-Agent", ConfigData.HeaderUserAgent.Value);
+                }
+
+                // Building login form data
+                var pairs = new Dictionary<string, string> {
+                { "username", ConfigData.Username.Value },
+                { "password", ConfigData.Password.Value },
+                { "submitme", "X" },
+                { "returnto", "%2F" }
             };
 
-            var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, null, true, null, LoginUrl);
+
-            await ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("Please wait..."), () =>
+                // Getting login form to retrieve PHPSESSID
+                var myRequest = new WebRequest()
                 {
-                    CQ dom = result.Content;
+                    Url = LoginUrl
-                    var errorMessage = dom[".tableinborder:eq(1) td"].Text().Trim();
+                };
-                    throw new ExceptionWithConfigData(errorMessage, configData);
+
+                // Add our headers to request
+                myRequest.Headers = emulatedBrowserHeaders;
+
+                // Get login page
+                var loginPage = await webclient.GetString(myRequest);
+
+                // Do the login
+                var request = new Utils.Clients.WebRequest()
+                {
+                    Cookies = loginPage.Cookies,
+                    PostData = pairs,
+                    Referer = LoginUrl,
+                    Type = RequestType.POST,
+                    Url = LoginCheckUrl,
+                    Headers = emulatedBrowserHeaders
+                };
+
+                // Perform loggin
+                latencyNow();
+                output("\nPerform login.. with " + LoginCheckUrl);
+                // Get login page
+                var response = await webclient.GetString(request);
+                output("\nTesting if we are logged...");
+                await ConfigureIfOK(response.Cookies, response.Cookies.Contains("uid="), () =>
+                {
+                    output("-> Login Failed: Wrong username or Password");
+                    throw new ExceptionWithConfigData("Wrong username or Password", configData);
                 });
-            return IndexerConfigurationStatus.RequiresTesting;
+
+                output("-> Login Success");
+
+                return IndexerConfigurationStatus.RequiresTesting;
+            }
+            catch (Exception e)
+            {
+                IsConfigured = false;
+                Console.WriteLine("Exception thrown : {0}.", e.Message);
+                Console.WriteLine(e.StackTrace);
+                throw e;
+            }
         }
 
+        /// <summary>
+        /// Execute our search query
+        /// </summary>
+        /// <param name="query">Query</param>
+        /// <returns>Releases</returns>
         protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
         {
             var releases = new List<ReleaseInfo>();
-            var searchUrl = BrowseUrl;
+            var torrentRowList = new List<CQ>();
-            var searchString = query.GetQueryString();
+            var searchTerm = query.GetQueryString();
+            var searchUrl = SearchUrl;
+            //int nbResults = 0;
+            //int pageLinkCount = 0;
 
-            var cats = MapTorznabCapsToTrackers(query);
+            // Check cache first so we don't query the server (if search term used or not in dev mode)
-            string cat = "a1";
+            if (!DevMode && !string.IsNullOrEmpty(searchTerm))
-
-            if (cats.Count == 1)
             {
-                cat = cats[0];
+                lock (cache)
+                {
+                    // Remove old cache items
+                    CleanCache();
+
+                    // Search in cache
+                    var cachedResult = cache.Where(i => i.Query == searchTerm).FirstOrDefault();
+                    if (cachedResult != null)
+                        return cachedResult.Results.Select(s => (ReleaseInfo)s.Clone()).ToArray();
+                }
             }
 
-            searchUrl += string.Format("?search={0}&param_val=0&complex_search=0&incldead={1}&orderby=added&sort=desc", HttpUtility.UrlEncode(searchString), cat);
+            // Build our query
+            var request = buildQuery(searchTerm, query, searchUrl);
+
+            // Getting results & Store content
+            fDom = await queryExec(request);
 
-            var response = await RequestStringWithCookiesAndRetry(searchUrl, null, BrowseUrl);
-            var results = response.Content;
             try
             {
-                CQ dom = results;
+                // Find number of results
-                var rows = dom["table#torrenttable > tbody > tr:has(td.tablea), table#torrents_table > tbody > tr#torrent-row"]; // selector for old and new style
+                //nbResults = ParseUtil.CoerceInt(Regex.Match(fDom["div.ajaxtotaltorrentcount"].Text(), @"\d+").Value);
 
-                foreach (var row in rows)
+                // Find torrent rows
+                var firstPageRows = findTorrentRows();
+
+                // Add them to torrents list
+                torrentRowList.AddRange(firstPageRows.Select(fRow => fRow.Cq()));
+
+                // Check if there are pagination links at bottom
+                /*Boolean pagination = (nbResults != 0);
+
+                // If pagination available
+                if (pagination)
                 {
-                    var release = new ReleaseInfo();
+                    // Calculate numbers of pages available for this search query (Based on number results and number of torrents on first page)
-                    var qRow = row.Cq();
+                    pageLinkCount = (int)Math.Ceiling((double)nbResults / firstPageRows.Length);
-                    var qTitleLink = qRow.Find("a[href^=\"details\"]").First();
+                }
-                    if (qTitleLink.HasAttr("title"))
+                else
-                        release.Title = qTitleLink.Attr("title");
+                {
+                    // Check if we have a minimum of one result
+                    if (firstPageRows.Length >= 1)
+                    {
+                        // Set page count arbitrary to one
+                        pageLinkCount = 1;
+                    }
                     else
-                        release.Title = qTitleLink.Text();
+                    {
-                    release.Description = qRow.Find("small > i").Text();
+                        output("\nNo result found for your query, please try another search term ...\n", "info");
-                    release.Guid = new Uri(SiteLink + qTitleLink.Attr("href"));
+                        // No result found for this query
-                    release.Comments = release.Guid;
+                        return releases;
+                    }
+                }
+                output("\nFound " + nbResults + " result(s) in " + pageLinkCount + " page(s) for this query !");
+                output("\nThere are " + firstPageRows.Length + " results on the first page !");
 
-                    // date is only available with Complex listing
+                // If we have a term used for search and pagination result superior to one
-                    var dateStr = qRow.Find("table > tbody > tr:nth-child(2) > td:nth-child(5)").Html().Replace("&nbsp;", " ");
+                if (!string.IsNullOrWhiteSpace(query.GetQueryString()) && pageLinkCount > 1)
-                    if (!string.IsNullOrEmpty(dateStr))
+                {
-                        release.PublishDate = DateTime.ParseExact(dateStr + " +0200", "dd.MM.yyyy HH:mm:ss zzz", CultureInfo.InvariantCulture);
+                    // Starting with page #2
+                    for (int i = 2; i <= Math.Min(Int32.Parse(ConfigData.Pages.Value), pageLinkCount); i++)
+                    {
+                        output("\nProcessing page #" + i);
 
-                    var qLink = qRow.Find("a[href^=\"download/\"]");
+                        // Request our page
-                    release.Link = new Uri(SiteLink + qLink.Attr("href"));
+                        latencyNow();
 
-                    var sizeStr = qRow.Find("td[nowrap]:nth-child(3), table > tbody > tr:nth-child(2) > td:nth-child(1) > b").Text().Trim();
+                        // Build our query
-                    release.Size = ReleaseInfo.GetBytes(sizeStr.Replace(".", "").Replace(",", "."));
+                        var pageRequest = buildQuery(searchTerm, query, searchUrl, i);
 
-                    release.Seeders = ParseUtil.CoerceInt(qRow.Find("td > a[href*=\"&toseeders=1\"]:first-child, td:has(a[href*=\"&toseeders=1\"]) > b:nth-child(1)").Text());
+                        // Getting results & Store content
-                    release.Peers = ParseUtil.CoerceInt(qRow.Find("td > a[href*=\"&todlers=1\"]:last-child, a[href*=\"&toseeders=1\"] + b").Text().Replace("L.", "")) + release.Seeders;
+                        fDom = await queryExec(pageRequest);
-                    release.Grabs = ParseUtil.CoerceLong(qRow.Find("td[style]:has(a[href*=\"tosnatchers=1\"])").Text().Replace(" Completed", ""));
 
-                    release.DownloadVolumeFactor = 0;
+                        // Process page results
-                    release.UploadVolumeFactor = 1;
+                        var additionalPageRows = findTorrentRows();
 
-                    var catLink = qRow.Find("a[onclick^=\"bparam(\"][onclick*=\"cat\"]");
+                        // Add them to torrents list
-                    var catId = catLink.Attr("onclick").Split('=')[1].Replace("');", "");
+                        torrentRowList.AddRange(additionalPageRows.Select(fRow => fRow.Cq()));
-                    if (!catId.StartsWith("scat"))
+                    }
-                        catId = "mc" + catId;
+                }*/
-                    release.Category = MapTrackerCatToNewznab(catId);
 
+                // Loop on results
+                foreach (CQ tRow in torrentRowList)
+                {
+                    output("\n=>> Torrent #" + (releases.Count + 1));
+
+                    // Release Name
+                    string name = tRow.Find("td:nth-child(2) > a > b").Text();
+                    output("Release: " + name);
+
+                    // Category
+                    string categoryID = tRow.Find("td:nth-child(1) > a").Attr("href").Split('=').Last();
+                    string categoryName = tRow.Find("td:nth-child(1) > a > img").Attr("alt");
+                    output("Category: " + MapTrackerCatToNewznab(categoryID).First().ToString() + " (" + categoryName + ")");
+
+                    // Uploader
+                    string uploader = tRow.Find("td.tt_row > span").Text();
+                    output("Uploader: " + uploader);
+
+                    // Seeders
+                    int seeders = ParseUtil.CoerceInt(Regex.Match(tRow.Find("td:nth-child(9)").Text().Trim(), @"\d+").Value);
+                    output("Seeders: " + seeders);
+
+                    // Leechers
+                    int leechers = ParseUtil.CoerceInt(Regex.Match(tRow.Find("td:nth-child(10)").Text().Trim(), @"\d+").Value);
+                    output("Leechers: " + leechers);
+
+                    // Completed
+                    int completed = ParseUtil.CoerceInt(Regex.Match(tRow.Find("td:nth-child(8)").Text().Trim(), @"\d+").Value);
+                    output("Completed: " + completed);
+
+                    // Comments
+                    int comments = ParseUtil.CoerceInt(Regex.Match(tRow.Find("td:nth-child(5)").Text().Trim(), @"\d+").Value);
+                    output("Comments: " + comments);
+
+                    // Files
+                    int files = ParseUtil.CoerceInt(Regex.Match(tRow.Find("td:nth-child(4)").Text().Trim(), @"\d+").Value);
+                    output("Files: " + files);
+
+                    // Size
+                    string humanSize = tRow.Find("td:nth-child(7)").Text().Trim();
+                    var size = ReleaseInfo.GetBytes(humanSize);
+                    output("Size: " + humanSize + " (" + size + " bytes)");
+
+                    // Publish Date
+                    var clock = DateTime.ParseExact(tRow.Find("td:nth-child(6) > nobr").Text().Trim(), "dd-MM-yyyyHH:mm:ss", CultureInfo.InvariantCulture);
+                    output("Released on: " + clock.ToShortDateString());
+
+                    // Torrent Details URL
+                    string details = tRow.Find("td:nth-child(2) > a").Attr("href").ToString().TrimStart('/');
+                    Uri detailsLink = new Uri(SiteLink + details);
+                    output("Details: " + detailsLink.AbsoluteUri);
+
+                    // Torrent Comments URL
+                    Uri commentsLink = new Uri(SiteLink + details + "&hit=1&tocomm=1");
+                    output("Comments Link: " + commentsLink.AbsoluteUri);
+
+                    // Torrent Download URL
+                    string download = tRow.Find("td:nth-child(3) > a").Attr("href");
+                    Uri downloadLink = new Uri(SiteLink + download);
+                    output("Download Link: " + downloadLink.AbsoluteUri);
+
+                    // Freeleech
+                    int downloadVolumeFactor = 1;
+                    if (tRow.Find("img[title^=\"FreeLeech\"]").Length >= 1)
+                    {
+                        downloadVolumeFactor = 0;
+                        output("FreeLeech =)");
+                    }
+
+                    // Building release infos
+                    var release = new ReleaseInfo()
+                    {
+                        Category = MapTrackerCatToNewznab(categoryID),
+                        Title = name,
+                        Seeders = seeders,
+                        Peers = seeders + leechers,
+                        Grabs = completed,
+                        MinimumRatio = 1,
+                        MinimumSeedTime = 345600,
+                        PublishDate = clock,
+                        Files = files,
+                        Size = size,
+                        Guid = detailsLink,
+                        Comments = commentsLink,
+                        Link = downloadLink,
+                        UploadVolumeFactor = 1,
+                        DownloadVolumeFactor = downloadVolumeFactor
+                    };
                     releases.Add(release);
                 }
+
             }
             catch (Exception ex)
             {
-                OnParseError(results, ex);
+                OnParseError("Error, unable to parse result \n" + ex.StackTrace, ex);
             }
 
+            // Return found releases
             return releases;
         }
+
+        /// <summary>
+        /// Build query to process
+        /// </summary>
+        /// <param name="term">Term to search</param>
+        /// <param name="query">Torznab Query for categories mapping</param>
+        /// <param name="url">Search url for provider</param>
+        /// <param name="page">Page number to request</param>
+        /// <returns>URL to query for parsing and processing results</returns>
+        private string buildQuery(string term, TorznabQuery query, string url, int page = 1)
+        {
+            var queryCollection = new NameValueCollection();
+
+            // Check if we are processing a new page
+            /*if (page > 1)
+            {
+                // Adding page number to query
+                url += "/" + page.ToString();
+            }*/
+
+            // If search term provided
+            if (!string.IsNullOrWhiteSpace(term))
+            {
+                // Add search term
+                queryCollection.Add("search", HttpUtility.UrlEncode(term));
+                queryCollection.Add("searchin", "0");
+                queryCollection.Add("cat", "0");
+                queryCollection.Add("incldead", "0");
+
+                // Building our query 
+                url += "?" + queryCollection.GetQueryString();
+            }
+            else
+            {
+                // Showing all torrents (just for output function)
+                term = "all";
+            }
+
+            output("\nBuilded query for \"" + term + "\"... " + url);
+
+            // Return our search url
+            return url;
+        }
+
+        /// <summary>
+        /// Switch Method for Querying
+        /// </summary>
+        /// <param name="request">URL created by Query Builder</param>
+        /// <returns>Results from query</returns>
+        private async Task<String> queryExec(string request)
+        {
+            String results = null;
+
+            // Switch in we are in DEV mode with Hard Drive Cache or not
+            if (DevMode && CacheMode)
+            {
+                // Check Cache before querying and load previous results if available
+                results = await queryCache(request);
+            }
+            else
+            {
+                // Querying tracker directly
+                results = await queryTracker(request);
+            }
+            return results;
+        }
+
+        /// <summary>
+        /// Get Torrents Page from Cache by Query Provided
+        /// </summary>
+        /// <param name="request">URL created by Query Builder</param>
+        /// <returns>Results from query</returns>
+        private async Task<String> queryCache(string request)
+        {
+            String results;
+
+            // Create Directory if not exist
+            System.IO.Directory.CreateDirectory(Directory);
+
+            // Clean Storage Provider Directory from outdated cached queries
+            cleanCacheStorage();
+
+            // File Name
+            string fileName = StringUtil.HashSHA1(request) + ".json";
+
+            // Create fingerprint for request
+            string file = Path.Combine(Directory, fileName);
+
+            // Checking modes states
+            if (File.Exists(file))
+            {
+                // File exist... loading it right now !
+                output("Loading results from hard drive cache ..." + fileName);
+                try
+                {
+                    using (StreamReader fileReader = File.OpenText(file))
+                    {
+                        JsonSerializer serializer = new JsonSerializer();
+                        results = (String)serializer.Deserialize(fileReader, typeof(String));
+                    }
+                }
+                catch (Exception e)
+                {
+                    output("Error loading cached results ! " + e.Message, "error");
+                    results = null;
+                }
+            }
+            else
+            {
+                // No cached file found, querying tracker directly
+                results = await queryTracker(request);
+
+                // Cached file didn't exist for our query, writing it right now !
+                output("Writing results to hard drive cache ..." + fileName);
+                using (StreamWriter fileWriter = File.CreateText(file))
+                {
+                    JsonSerializer serializer = new JsonSerializer();
+                    serializer.Serialize(fileWriter, results);
+                }
+                output("Writed to " + Directory + fileName);
+            }
+            return results;
+        }
+
+        /// <summary>
+        /// Get Torrents Page from Tracker by Query Provided
+        /// </summary>
+        /// <param name="request">URL created by Query Builder</param>
+        /// <returns>Results from query</returns>
+        private async Task<String> queryTracker(string request)
+        {
+            WebClientStringResult results = null;
+
+            // Cache mode not enabled or cached file didn't exist for our query
+            output("\nQuerying tracker for results....");
+
+            // Request our first page
+            latencyNow();
+            results = await RequestStringWithCookiesAndRetry(request, null, null, emulatedBrowserHeaders);
+
+            // Return results from tracker
+            return results.Content;
+        }
+
+        /// <summary>
+        /// Clean Hard Drive Cache Storage
+        /// </summary>
+        /// <param name="force">Force Provider Folder deletion</param>
+        private void cleanCacheStorage(bool force = false)
+        {
+            // Check cleaning method
+            if (force)
+            {
+                // Deleting Provider Storage folder and all files recursively
+                output("\nDeleting Provider Storage folder and all files recursively ...");
+
+                // Check if directory exist
+                if (System.IO.Directory.Exists(Directory))
+                {
+                    // Delete storage directory of provider
+                    System.IO.Directory.Delete(Directory, true);
+                    output("-> Storage folder deleted successfully.");
+                }
+                else
+                {
+                    // No directory, so nothing to do
+                    output("-> No Storage folder found for this provider !");
+                }
+            }
+            else
+            {
+                var i = 0;
+                // Check if there is file older than ... and delete them
+                output("\nCleaning Provider Storage folder... in progress.");
+                System.IO.Directory.GetFiles(Directory)
+                .Select(f => new FileInfo(f))
+                .Where(f => f.LastAccessTime < DateTime.Now.AddMilliseconds(-Convert.ToInt32(ConfigData.HardDriveCacheKeepTime.Value)))
+                .ToList()
+                .ForEach(f =>
+                {
+                    output("Deleting cached file << " + f.Name + " >> ... done.");
+                    f.Delete();
+                    i++;
+                });
+
+                // Inform on what was cleaned during process
+                if (i > 0)
+                {
+                    output("-> Deleted " + i + " cached files during cleaning.");
+                }
+                else
+                {
+                    output("-> Nothing deleted during cleaning.");
+                }
+            }
+        }
+
+        /// <summary>
+        /// Generate a random fake latency to avoid detection on tracker side
+        /// </summary>
+        private void latencyNow()
+        {
+            // Need latency ?
+            if (Latency)
+            {
+                // Generate a random value in our range
+                var random = new Random(DateTime.Now.Millisecond);
+                int waiting = random.Next(Convert.ToInt32(ConfigData.LatencyStart.Value), Convert.ToInt32(ConfigData.LatencyEnd.Value));
+                output("\nLatency Faker => Sleeping for " + waiting + " ms...");
+
+                // Sleep now...
+                System.Threading.Thread.Sleep(waiting);
+            }
+        }
+
+        /// <summary>
+        /// Find torrent rows in search pages
+        /// </summary>
+        /// <returns>JQuery Object</returns>
+        private CQ findTorrentRows()
+        {
+            // Return all occurencis of torrents found
+            return fDom["#minions .browse,.sticky"];
+        }
+
+        /// <summary>
+        /// Output message for logging or developpment (console)
+        /// </summary>
+        /// <param name="message">Message to output</param>
+        /// <param name="level">Level for Logger</param>
+        private void output(string message, string level = "debug")
+        {
+            // Check if we are in dev mode
+            if (DevMode)
+            {
+                // Output message to console
+                Console.WriteLine(message);
+            }
+            else
+            {
+                // Send message to logger with level
+                switch (level)
+                {
+                    default:
+                        goto case "debug";
+                    case "debug":
+                        // Only if Debug Level Enabled on Jackett
+                        if (Engine.Logger.IsDebugEnabled)
+                        {
+                            logger.Debug(message);
+                        }
+                        break;
+                    case "info":
+                        logger.Info(message);
+                        break;
+                    case "error":
+                        logger.Error(message);
+                        break;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Validate Config entered by user on Jackett
+        /// </summary>
+        private void validateConfig()
+        {
+            output("\nValidating Settings ... \n");
+
+            // Check Username Setting
+            if (string.IsNullOrEmpty(ConfigData.Username.Value))
+            {
+                throw new ExceptionWithConfigData("You must provide a username for this tracker to login !", ConfigData);
+            }
+            else
+            {
+                output("Validated Setting -- Username (auth) => " + ConfigData.Username.Value.ToString());
+            }
+
+            // Check Password Setting
+            if (string.IsNullOrEmpty(ConfigData.Password.Value))
+            {
+                throw new ExceptionWithConfigData("You must provide a password with your username for this tracker to login !", ConfigData);
+            }
+            else
+            {
+                output("Validated Setting -- Password (auth) => " + ConfigData.Password.Value.ToString());
+            }
+
+            // Check Max Page Setting
+            /*if (!string.IsNullOrEmpty(ConfigData.Pages.Value))
+            {
+                try
+                {
+                    output("Validated Setting -- Max Pages => " + Convert.ToInt32(ConfigData.Pages.Value));
+                }
+                catch (Exception)
+                {
+                    throw new ExceptionWithConfigData("Please enter a numeric maximum number of pages to crawl !", ConfigData);
+                }
+            }
+            else
+            {
+                throw new ExceptionWithConfigData("Please enter a maximum number of pages to crawl !", ConfigData);
+            }*/
+
+            // Check Latency Setting
+            if (ConfigData.Latency.Value)
+            {
+                output("\nValidated Setting -- Latency Simulation enabled");
+
+                // Check Latency Start Setting
+                if (!string.IsNullOrEmpty(ConfigData.LatencyStart.Value))
+                {
+                    try
+                    {
+                        output("Validated Setting -- Latency Start => " + Convert.ToInt32(ConfigData.LatencyStart.Value));
+                    }
+                    catch (Exception)
+                    {
+                        throw new ExceptionWithConfigData("Please enter a numeric latency start in ms !", ConfigData);
+                    }
+                }
+                else
+                {
+                    throw new ExceptionWithConfigData("Latency Simulation enabled, Please enter a start latency !", ConfigData);
+                }
+
+                // Check Latency End Setting
+                if (!string.IsNullOrEmpty(ConfigData.LatencyEnd.Value))
+                {
+                    try
+                    {
+                        output("Validated Setting -- Latency End => " + Convert.ToInt32(ConfigData.LatencyEnd.Value));
+                    }
+                    catch (Exception)
+                    {
+                        throw new ExceptionWithConfigData("Please enter a numeric latency end in ms !", ConfigData);
+                    }
+                }
+                else
+                {
+                    throw new ExceptionWithConfigData("Latency Simulation enabled, Please enter a end latency !", ConfigData);
+                }
+            }
+
+            // Check Browser Setting
+            if (ConfigData.Browser.Value)
+            {
+                output("\nValidated Setting -- Browser Simulation enabled");
+
+                // Check ACCEPT header Setting
+                if (string.IsNullOrEmpty(ConfigData.HeaderAccept.Value))
+                {
+                    throw new ExceptionWithConfigData("Browser Simulation enabled, Please enter an ACCEPT header !", ConfigData);
+                }
+                else
+                {
+                    output("Validated Setting -- ACCEPT (header) => " + ConfigData.HeaderAccept.Value.ToString());
+                }
+
+                // Check ACCEPT-LANG header Setting
+                if (string.IsNullOrEmpty(ConfigData.HeaderAcceptLang.Value))
+                {
+                    throw new ExceptionWithConfigData("Browser Simulation enabled, Please enter an ACCEPT-LANG header !", ConfigData);
+                }
+                else
+                {
+                    output("Validated Setting -- ACCEPT-LANG (header) => " + ConfigData.HeaderAcceptLang.Value.ToString());
+                }
+
+                // Check USER-AGENT header Setting
+                if (string.IsNullOrEmpty(ConfigData.HeaderUserAgent.Value))
+                {
+                    throw new ExceptionWithConfigData("Browser Simulation enabled, Please enter an USER-AGENT header !", ConfigData);
+                }
+                else
+                {
+                    output("Validated Setting -- USER-AGENT (header) => " + ConfigData.HeaderUserAgent.Value.ToString());
+                }
+            }
+
+            // Check Dev Cache Settings
+            if (ConfigData.HardDriveCache.Value == true)
+            {
+                output("\nValidated Setting -- DEV Hard Drive Cache enabled");
+
+                // Check if Dev Mode enabled !
+                if (!ConfigData.DevMode.Value)
+                {
+                    throw new ExceptionWithConfigData("Hard Drive is enabled but not in DEV MODE, Please enable DEV MODE !", ConfigData);
+                }
+
+                // Check Cache Keep Time Setting
+                if (!string.IsNullOrEmpty(ConfigData.HardDriveCacheKeepTime.Value))
+                {
+                    try
+                    {
+                        output("Validated Setting -- Cache Keep Time (ms) => " + Convert.ToInt32(ConfigData.HardDriveCacheKeepTime.Value));
+                    }
+                    catch (Exception)
+                    {
+                        throw new ExceptionWithConfigData("Please enter a numeric hard drive keep time in ms !", ConfigData);
+                    }
+                }
+                else
+                {
+                    throw new ExceptionWithConfigData("Hard Drive Cache enabled, Please enter a maximum keep time for cache !", ConfigData);
+                }
+            }
+            else
+            {
+                // Delete cache if previously existed
+                cleanCacheStorage(true);
+            }
+        }
     }
 }
-

src/Jackett/Jackett.csproj

@@ -268,7 +268,6 @@
     <Compile Include="Indexers\Pretome.cs" />
     <Compile Include="Indexers\PrivateHD.cs" />
     <Compile Include="Indexers\SceneAccess.cs" />
-    <Compile Include="Indexers\SceneFZ.cs" />
     <Compile Include="Indexers\SceneTime.cs" />
     <Compile Include="Indexers\SpeedCD.cs" />
     <Compile Include="Indexers\TorrentDay.cs" />
@@ -382,6 +381,8 @@
     <Compile Include="Models\DTO\TorrentPotatoResponse.cs" />
     <Compile Include="Models\DTO\TorrentPotatoResponseItem.cs" />
     <Compile Include="Models\DTO\TorznabRequest.cs" />
+    <Compile Include="Indexers\SceneFZ.cs" />
+    <Compile Include="Models\IndexerConfig\Bespoke\ConfigurationDataSceneFZ.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="App.config">

src/Jackett/Models/IndexerConfig/Bespoke/ConfigurationDataSceneFZ.cs

@@ -0,0 +1,55 @@
+namespace Jackett.Models.IndexerConfig.Bespoke
+{
+    class ConfigurationDataSceneFZ : ConfigurationData
+    {
+        public DisplayItem CredentialsWarning { get; private set; }
+        public StringItem Username { get; private set; }
+        public StringItem Password { get; private set; }
+        public DisplayItem PagesWarning { get; private set; }
+        //public StringItem Pages { get; private set; }
+        public BoolItem Dead { get; private set; }
+        public DisplayItem SecurityWarning { get; private set; }
+        public BoolItem Latency { get; private set; }
+        public BoolItem Browser { get; private set; }
+        public DisplayItem LatencyWarning { get; private set; }
+        public StringItem LatencyStart { get; private set; }
+        public StringItem LatencyEnd { get; private set; }
+        public DisplayItem HeadersWarning { get; private set; }
+        public StringItem HeaderAccept { get; private set; }
+        public StringItem HeaderAcceptLang { get; private set; }
+        public BoolItem HeaderDNT { get; private set; }
+        public BoolItem HeaderUpgradeInsecure { get; private set; }
+        public StringItem HeaderUserAgent { get; private set; }
+        public DisplayItem DevWarning { get; private set; }
+        public BoolItem DevMode { get; private set; }
+        public BoolItem HardDriveCache { get; private set; }
+        public StringItem HardDriveCacheKeepTime { get; private set; }
+
+        public ConfigurationDataSceneFZ()
+            : base()
+        {
+            CredentialsWarning = new DisplayItem("<b>Credentials Configuration</b> (<i>Private Tracker</i>),<br /><br /> <ul><li><b>Username</b> is your account name on this tracker.</li><li><b>Password</b> is your password associated to your account name.</li></ul>") { Name = "Credentials" };
+            Username = new StringItem { Name = "Username (Required)", Value = "" };
+            Password = new StringItem { Name = "Password (Required)", Value = "" };
+            PagesWarning = new DisplayItem("<b>Preferences Configuration</b> (<i>Tweak your search settings</i>),<br /><br /> <ul><li><b>Include Dead</b> let you search <u>including</u> torrents which are marked as Dead.</li></ul>") { Name = "Preferences" };
+            //Pages = new StringItem { Name = "Max Pages to Process (Required)", Value = "4" };
+            Dead = new BoolItem() { Name = "Include Dead (Optional)", Value = false };
+            SecurityWarning = new DisplayItem("<b>Security Configuration</b> (<i>Read this area carefully !</i>),<br /><br /> <ul><li><b>Latency Simulation</b> will simulate human browsing with Jacket by pausing Jacket for an random time between each request, to fake a real content browsing.</li><li><b>Browser Simulation</b> will simulate a real human browser by injecting additionals headers when doing requests to tracker.</li></ul>") { Name = "Security" };
+            Latency = new BoolItem() { Name = "Latency Simulation (Optional)", Value = false };
+            Browser = new BoolItem() { Name = "Browser Simulation (Optional)", Value = false };
+            LatencyWarning = new DisplayItem("<b>Latency Configuration</b> (<i>Required if latency simulation enabled</i>),<br /><br/> <ul><li>By filling this range, <b>Jackett will make a random timed pause</b> <u>between requests</u> to tracker <u>to simulate a real browser</u>.</li><li>MilliSeconds <b>only</b></li></ul>") { Name = "Simulate Latency" };
+            LatencyStart = new StringItem { Name = "Minimum Latency (ms)", Value = "1589" };
+            LatencyEnd = new StringItem { Name = "Maximum Latency (ms)", Value = "3674" };
+            HeadersWarning = new DisplayItem("<b>Browser Headers Configuration</b> (<i>Required if browser simulation enabled</i>),<br /><br /> <ul><li>By filling these fields, <b>Jackett will inject headers</b> with your values <u>to simulate a real browser</u>.</li><li>You can get <b>your browser values</b> here: <a href='https://www.whatismybrowser.com/detect/what-http-headers-is-my-browser-sending' target='blank'>www.whatismybrowser.com</a></li></ul><br /><i><b>Note that</b> some headers are not necessary because they are injected automatically by this provider such as Accept_Encoding, Connection, Host or X-Requested-With</i>") { Name = "Injecting headers" };
+            HeaderAccept = new StringItem { Name = "Accept", Value = "" };
+            HeaderAcceptLang = new StringItem { Name = "Accept-Language", Value = "" };
+            HeaderDNT = new BoolItem { Name = "DNT", Value = false };
+            HeaderUpgradeInsecure = new BoolItem { Name = "Upgrade-Insecure-Requests", Value = false };
+            HeaderUserAgent = new StringItem { Name = "User-Agent", Value = "" };
+            DevWarning = new DisplayItem("<b>Development Facility</b> (<i>For Developers ONLY</i>),<br /><br /> <ul><li>By enabling development mode, <b>Jackett will bypass his cache</b> and will <u>output debug messages to console</u> instead of his log file.</li><li>By enabling Hard Drive Cache, <b>This provider</b> will <u>save each query answers from tracker</u> in temp directory, in fact this reduce drastically HTTP requests when building a provider at parsing step for example. So, <b> Jackett will search for a cached query answer on hard drive before executing query on tracker side !</b> <i>DEV MODE must be enabled to use it !</li></ul>") { Name = "Development" };
+            DevMode = new BoolItem { Name = "Enable DEV MODE (Developers ONLY)", Value = true };
+            HardDriveCache = new BoolItem { Name = "Enable HARD DRIVE CACHE (Developers ONLY)", Value = true };
+            HardDriveCacheKeepTime = new StringItem { Name = "Keep Cached files for (ms)", Value = "300000" };
+        }
+    }
+}