diff --git a/src/Jackett/Indexers/SceneFZ.cs b/src/Jackett/Indexers/SceneFZ.cs index 7d0c86b19..bf94db6a7 100644 --- a/src/Jackett/Indexers/SceneFZ.cs +++ b/src/Jackett/Indexers/SceneFZ.cs @@ -1,186 +1,801 @@ -using Jackett.Utils.Clients; -using NLog; +using System; +using System.Collections.Generic; +using System.Collections.Specialized; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using System.Web; +using CsQuery; +using Jackett.Models; +using Jackett.Models.IndexerConfig.Bespoke; using Jackett.Services; using Jackett.Utils; -using Jackett.Models; -using System.Threading.Tasks; +using Jackett.Utils.Clients; +using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using System.Collections.Generic; -using CsQuery; -using System.Web; -using System; -using System.Text; -using System.Globalization; -using Jackett.Models.IndexerConfig; +using NLog; namespace Jackett.Indexers { - public class SceneFZ : BaseWebIndexer + /// + /// Provider for SceneFZ + /// + public class SceneFZ : BaseCachingWebIndexer { - string LoginUrl { get { return SiteLink + "takelogin.php"; } } + private string LoginUrl { get { return SiteLink + "login.php"; } } + private string LoginCheckUrl { get { return SiteLink + "takelogin.php"; } } + private string SearchUrl { get { return SiteLink + "torrents.php"; } } + private bool Latency { get { return ConfigData.Latency.Value; } } + private bool DevMode { get { return ConfigData.DevMode.Value; } } + private bool CacheMode { get { return ConfigData.HardDriveCache.Value; } } + private static string Directory => Path.Combine(Path.GetTempPath(), Assembly.GetExecutingAssembly().GetName().Name.ToLower(), MethodBase.GetCurrentMethod().DeclaringType?.Name.ToLower()); - string BrowseUrl { get { return SiteLink + "ajax_browse.php"; } } + private Dictionary emulatedBrowserHeaders = new Dictionary(); + private CQ fDom = null; - new ConfigurationDataBasicLogin configData + private ConfigurationDataSceneFZ ConfigData { - get { return (ConfigurationDataBasicLogin)base.configData; } + get { return (ConfigurationDataSceneFZ)configData; } set { base.configData = value; } } - public SceneFZ(IIndexerConfigurationService configService, IWebClient wc, Logger l, IProtectionService ps) - : base(name: "SceneFZ", - description: "Torrent tracker. Tracking over 50.000 torrent files.", - link: "http://scenefz.me/", - caps: TorznabUtil.CreateDefaultTorznabTVCaps(), - configService: configService, - client: wc, - logger: l, - p: ps, - configData: new ConfigurationDataBasicLogin()) + public SceneFZ(IIndexerConfigurationService configService, IWebClient w, Logger l, IProtectionService ps) + : base( + name: "SceneFZ", + description: "Torrent tracker. Tracking over 50.000 torrent files.", + link: "https://scenefz.me/", + caps: TorznabUtil.CreateDefaultTorznabTVCaps(), + configService: configService, + client: w, + logger: l, + p: ps, + configData: new ConfigurationDataSceneFZ()) { Encoding = Encoding.UTF8; Language = "ro-ro"; Type = "private"; - this.configData.Instructions.Value = "The published date is only available if you set \"Torrent Listing\" to Complex is your profile."; + // Clean capabilities + TorznabCaps.Categories.Clear(); - AddCategoryMapping("mc32", TorznabCatType.Movies); // Movies - AddCategoryMapping("scat22", TorznabCatType.MoviesBluRay); // BluRay - AddCategoryMapping("scat40", TorznabCatType.MoviesSD); // Xvid - AddCategoryMapping("scat41", TorznabCatType.MoviesDVD); // Dvd - AddCategoryMapping("scat47", TorznabCatType.MoviesHD); // HD - AddCategoryMapping("scat52", TorznabCatType.MoviesDVD); // DVD-RO - AddCategoryMapping("scat53", TorznabCatType.MoviesHD); // HD-RO - AddCategoryMapping("scat54", TorznabCatType.MoviesBluRay); // BluRay-RO - AddCategoryMapping("scat55", TorznabCatType.MoviesSD); // XVID-RO - AddCategoryMapping("scat60", TorznabCatType.MoviesOther); // Sport - AddCategoryMapping("mc33", TorznabCatType.TV); // TV - AddCategoryMapping("scat66", TorznabCatType.TVSD); // SD - AddCategoryMapping("scat67", TorznabCatType.TVSD); // SD-RO - AddCategoryMapping("scat68", TorznabCatType.TVHD); // HD - AddCategoryMapping("scat69", TorznabCatType.TVHD); // HDTV-RO - AddCategoryMapping("mc30", TorznabCatType.Console); // Games - AddCategoryMapping("scat58", TorznabCatType.ConsolePS3); // PS2 - AddCategoryMapping("scat16", TorznabCatType.PCGames); // Pc-Iso - AddCategoryMapping("scat17", TorznabCatType.Console); // Misc - AddCategoryMapping("scat18", TorznabCatType.PCGames); // Pc-Rip - AddCategoryMapping("scat19", TorznabCatType.Console); // Consoles - AddCategoryMapping("scat57", TorznabCatType.ConsoleXbox360); // Xbox 360 - AddCategoryMapping("scat46", TorznabCatType.Console); // Oldies - AddCategoryMapping("scat59", TorznabCatType.ConsolePS3); // PS3 - AddCategoryMapping("mc31", TorznabCatType.PC); // Soft - AddCategoryMapping("scat20", TorznabCatType.PC); // Pc-Iso - AddCategoryMapping("scat21", TorznabCatType.PC); // Misc - AddCategoryMapping("scat48", TorznabCatType.PCMac); // Mac OS - AddCategoryMapping("mc27", TorznabCatType.Audio); // Music - AddCategoryMapping("scat8", TorznabCatType.AudioMP3); // MP3 - AddCategoryMapping("scat45", TorznabCatType.AudioVideo); // Videoclips - AddCategoryMapping("scat61", TorznabCatType.AudioLossless); // FLAC - AddCategoryMapping("mc35", TorznabCatType.PCPhoneOther); // Mobile - AddCategoryMapping("scat44", TorznabCatType.PCPhoneOther); // Misc - AddCategoryMapping("scat64", TorznabCatType.PCPhoneIOS); // iOS - AddCategoryMapping("scat65", TorznabCatType.PCPhoneAndroid); // Android - AddCategoryMapping("mc28", TorznabCatType.TVAnime); // Anime - AddCategoryMapping("scat13", TorznabCatType.TVAnime); // Tv-Eps - AddCategoryMapping("scat12", TorznabCatType.TVAnime); // Cartoons - AddCategoryMapping("mc29", TorznabCatType.TVDocumentary); // Docs - AddCategoryMapping("scat14", TorznabCatType.Books); // Books - AddCategoryMapping("scat15", TorznabCatType.Other); // Misc - AddCategoryMapping("mc36", TorznabCatType.PC0day); // 0Day - AddCategoryMapping("mc34", TorznabCatType.XXX); // XXX 18 - AddCategoryMapping("scat33", TorznabCatType.Other); // Images - AddCategoryMapping("scat34", TorznabCatType.Other); // Video + AddCategoryMapping(1, TorznabCatType.TVAnime); + AddCategoryMapping(49, TorznabCatType.Other); + AddCategoryMapping(43, TorznabCatType.TVDocumentary); + AddCategoryMapping(35, TorznabCatType.Console); + AddCategoryMapping(30, TorznabCatType.PC); + AddCategoryMapping(13, TorznabCatType.TVHD); + AddCategoryMapping(14, TorznabCatType.TVHD); + AddCategoryMapping(44, TorznabCatType.Other); + AddCategoryMapping(45, TorznabCatType.PC); + AddCategoryMapping(15, TorznabCatType.PCPhoneOther); + AddCategoryMapping(61, TorznabCatType.Movies3D); + AddCategoryMapping(62, TorznabCatType.Movies3D); + AddCategoryMapping(5, TorznabCatType.MoviesBluRay); + AddCategoryMapping(6, TorznabCatType.MoviesBluRay); + AddCategoryMapping(9, TorznabCatType.MoviesDVD); + AddCategoryMapping(10, TorznabCatType.MoviesDVD); + AddCategoryMapping(11, TorznabCatType.MoviesHD); + AddCategoryMapping(12, TorznabCatType.MoviesHD); + AddCategoryMapping(24, TorznabCatType.MoviesSD); + AddCategoryMapping(25, TorznabCatType.MoviesSD); + AddCategoryMapping(27, TorznabCatType.XXX); + AddCategoryMapping(28, TorznabCatType.Audio); + AddCategoryMapping(64, TorznabCatType.AudioLossless); + AddCategoryMapping(29, TorznabCatType.AudioVideo); + AddCategoryMapping(26, TorznabCatType.PCISO); + AddCategoryMapping(22, TorznabCatType.TVSport); + AddCategoryMapping(20, TorznabCatType.TVSD); + AddCategoryMapping(21, TorznabCatType.TVSD); } + /// + /// Configure our SceneFZ Provider + /// + /// Our params in Json + /// Configuration state public override async Task ApplyConfiguration(JToken configJson) { - LoadValuesFromJson(configJson); - var pairs = new Dictionary + try { - { "username", configData.Username.Value }, - { "password", configData.Password.Value } + // Retrieve config values set by Jackett's user + LoadValuesFromJson(configJson); + + // Check & Validate Config + validateConfig(); + + // Setting our data for a better emulated browser (maximum security) + // TODO: Encoded Content not supported by Jackett at this time + // emulatedBrowserHeaders.Add("Accept-Encoding", "gzip, deflate"); + + // If we want to simulate a browser + if (ConfigData.Browser.Value) + { + + // Clean headers + emulatedBrowserHeaders.Clear(); + + // Inject headers + emulatedBrowserHeaders.Add("Accept", ConfigData.HeaderAccept.Value); + emulatedBrowserHeaders.Add("Accept-Language", ConfigData.HeaderAcceptLang.Value); + emulatedBrowserHeaders.Add("DNT", Convert.ToInt32(ConfigData.HeaderDNT.Value).ToString()); + emulatedBrowserHeaders.Add("Upgrade-Insecure-Requests", Convert.ToInt32(ConfigData.HeaderUpgradeInsecure.Value).ToString()); + emulatedBrowserHeaders.Add("User-Agent", ConfigData.HeaderUserAgent.Value); + } + + // Building login form data + var pairs = new Dictionary { + { "username", ConfigData.Username.Value }, + { "password", ConfigData.Password.Value }, + { "submitme", "X" }, + { "returnto", "%2F" } }; - var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, null, true, null, LoginUrl); - await ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("Please wait..."), () => + + // Getting login form to retrieve PHPSESSID + var myRequest = new WebRequest() { - CQ dom = result.Content; - var errorMessage = dom[".tableinborder:eq(1) td"].Text().Trim(); - throw new ExceptionWithConfigData(errorMessage, configData); + Url = LoginUrl + }; + + // Add our headers to request + myRequest.Headers = emulatedBrowserHeaders; + + // Get login page + var loginPage = await webclient.GetString(myRequest); + + // Do the login + var request = new Utils.Clients.WebRequest() + { + Cookies = loginPage.Cookies, + PostData = pairs, + Referer = LoginUrl, + Type = RequestType.POST, + Url = LoginCheckUrl, + Headers = emulatedBrowserHeaders + }; + + // Perform loggin + latencyNow(); + output("\nPerform login.. with " + LoginCheckUrl); + // Get login page + var response = await webclient.GetString(request); + output("\nTesting if we are logged..."); + await ConfigureIfOK(response.Cookies, response.Cookies.Contains("uid="), () => + { + output("-> Login Failed: Wrong username or Password"); + throw new ExceptionWithConfigData("Wrong username or Password", configData); }); - return IndexerConfigurationStatus.RequiresTesting; + + output("-> Login Success"); + + return IndexerConfigurationStatus.RequiresTesting; + } + catch (Exception e) + { + IsConfigured = false; + Console.WriteLine("Exception thrown : {0}.", e.Message); + Console.WriteLine(e.StackTrace); + throw e; + } } + /// + /// Execute our search query + /// + /// Query + /// Releases protected override async Task> PerformQuery(TorznabQuery query) { var releases = new List(); - var searchUrl = BrowseUrl; - var searchString = query.GetQueryString(); + var torrentRowList = new List(); + var searchTerm = query.GetQueryString(); + var searchUrl = SearchUrl; + //int nbResults = 0; + //int pageLinkCount = 0; - var cats = MapTorznabCapsToTrackers(query); - string cat = "a1"; - - if (cats.Count == 1) + // Check cache first so we don't query the server (if search term used or not in dev mode) + if (!DevMode && !string.IsNullOrEmpty(searchTerm)) { - cat = cats[0]; + lock (cache) + { + // Remove old cache items + CleanCache(); + + // Search in cache + var cachedResult = cache.Where(i => i.Query == searchTerm).FirstOrDefault(); + if (cachedResult != null) + return cachedResult.Results.Select(s => (ReleaseInfo)s.Clone()).ToArray(); + } } - searchUrl += string.Format("?search={0}¶m_val=0&complex_search=0&incldead={1}&orderby=added&sort=desc", HttpUtility.UrlEncode(searchString), cat); + // Build our query + var request = buildQuery(searchTerm, query, searchUrl); + + // Getting results & Store content + fDom = await queryExec(request); - var response = await RequestStringWithCookiesAndRetry(searchUrl, null, BrowseUrl); - var results = response.Content; try { - CQ dom = results; - var rows = dom["table#torrenttable > tbody > tr:has(td.tablea), table#torrents_table > tbody > tr#torrent-row"]; // selector for old and new style + // Find number of results + //nbResults = ParseUtil.CoerceInt(Regex.Match(fDom["div.ajaxtotaltorrentcount"].Text(), @"\d+").Value); - foreach (var row in rows) + // Find torrent rows + var firstPageRows = findTorrentRows(); + + // Add them to torrents list + torrentRowList.AddRange(firstPageRows.Select(fRow => fRow.Cq())); + + // Check if there are pagination links at bottom + /*Boolean pagination = (nbResults != 0); + + // If pagination available + if (pagination) { - var release = new ReleaseInfo(); - var qRow = row.Cq(); - var qTitleLink = qRow.Find("a[href^=\"details\"]").First(); - if (qTitleLink.HasAttr("title")) - release.Title = qTitleLink.Attr("title"); + // Calculate numbers of pages available for this search query (Based on number results and number of torrents on first page) + pageLinkCount = (int)Math.Ceiling((double)nbResults / firstPageRows.Length); + } + else + { + // Check if we have a minimum of one result + if (firstPageRows.Length >= 1) + { + // Set page count arbitrary to one + pageLinkCount = 1; + } else - release.Title = qTitleLink.Text(); - release.Description = qRow.Find("small > i").Text(); - release.Guid = new Uri(SiteLink + qTitleLink.Attr("href")); - release.Comments = release.Guid; + { + output("\nNo result found for your query, please try another search term ...\n", "info"); + // No result found for this query + return releases; + } + } + output("\nFound " + nbResults + " result(s) in " + pageLinkCount + " page(s) for this query !"); + output("\nThere are " + firstPageRows.Length + " results on the first page !"); - // date is only available with Complex listing - var dateStr = qRow.Find("table > tbody > tr:nth-child(2) > td:nth-child(5)").Html().Replace(" ", " "); - if (!string.IsNullOrEmpty(dateStr)) - release.PublishDate = DateTime.ParseExact(dateStr + " +0200", "dd.MM.yyyy HH:mm:ss zzz", CultureInfo.InvariantCulture); + // If we have a term used for search and pagination result superior to one + if (!string.IsNullOrWhiteSpace(query.GetQueryString()) && pageLinkCount > 1) + { + // Starting with page #2 + for (int i = 2; i <= Math.Min(Int32.Parse(ConfigData.Pages.Value), pageLinkCount); i++) + { + output("\nProcessing page #" + i); - var qLink = qRow.Find("a[href^=\"download/\"]"); - release.Link = new Uri(SiteLink + qLink.Attr("href")); + // Request our page + latencyNow(); - var sizeStr = qRow.Find("td[nowrap]:nth-child(3), table > tbody > tr:nth-child(2) > td:nth-child(1) > b").Text().Trim(); - release.Size = ReleaseInfo.GetBytes(sizeStr.Replace(".", "").Replace(",", ".")); + // Build our query + var pageRequest = buildQuery(searchTerm, query, searchUrl, i); - release.Seeders = ParseUtil.CoerceInt(qRow.Find("td > a[href*=\"&toseeders=1\"]:first-child, td:has(a[href*=\"&toseeders=1\"]) > b:nth-child(1)").Text()); - release.Peers = ParseUtil.CoerceInt(qRow.Find("td > a[href*=\"&todlers=1\"]:last-child, a[href*=\"&toseeders=1\"] + b").Text().Replace("L.", "")) + release.Seeders; - release.Grabs = ParseUtil.CoerceLong(qRow.Find("td[style]:has(a[href*=\"tosnatchers=1\"])").Text().Replace(" Completed", "")); + // Getting results & Store content + fDom = await queryExec(pageRequest); - release.DownloadVolumeFactor = 0; - release.UploadVolumeFactor = 1; + // Process page results + var additionalPageRows = findTorrentRows(); - var catLink = qRow.Find("a[onclick^=\"bparam(\"][onclick*=\"cat\"]"); - var catId = catLink.Attr("onclick").Split('=')[1].Replace("');", ""); - if (!catId.StartsWith("scat")) - catId = "mc" + catId; - release.Category = MapTrackerCatToNewznab(catId); + // Add them to torrents list + torrentRowList.AddRange(additionalPageRows.Select(fRow => fRow.Cq())); + } + }*/ + // Loop on results + foreach (CQ tRow in torrentRowList) + { + output("\n=>> Torrent #" + (releases.Count + 1)); + + // Release Name + string name = tRow.Find("td:nth-child(2) > a > b").Text(); + output("Release: " + name); + + // Category + string categoryID = tRow.Find("td:nth-child(1) > a").Attr("href").Split('=').Last(); + string categoryName = tRow.Find("td:nth-child(1) > a > img").Attr("alt"); + output("Category: " + MapTrackerCatToNewznab(categoryID).First().ToString() + " (" + categoryName + ")"); + + // Uploader + string uploader = tRow.Find("td.tt_row > span").Text(); + output("Uploader: " + uploader); + + // Seeders + int seeders = ParseUtil.CoerceInt(Regex.Match(tRow.Find("td:nth-child(9)").Text().Trim(), @"\d+").Value); + output("Seeders: " + seeders); + + // Leechers + int leechers = ParseUtil.CoerceInt(Regex.Match(tRow.Find("td:nth-child(10)").Text().Trim(), @"\d+").Value); + output("Leechers: " + leechers); + + // Completed + int completed = ParseUtil.CoerceInt(Regex.Match(tRow.Find("td:nth-child(8)").Text().Trim(), @"\d+").Value); + output("Completed: " + completed); + + // Comments + int comments = ParseUtil.CoerceInt(Regex.Match(tRow.Find("td:nth-child(5)").Text().Trim(), @"\d+").Value); + output("Comments: " + comments); + + // Files + int files = ParseUtil.CoerceInt(Regex.Match(tRow.Find("td:nth-child(4)").Text().Trim(), @"\d+").Value); + output("Files: " + files); + + // Size + string humanSize = tRow.Find("td:nth-child(7)").Text().Trim(); + var size = ReleaseInfo.GetBytes(humanSize); + output("Size: " + humanSize + " (" + size + " bytes)"); + + // Publish Date + var clock = DateTime.ParseExact(tRow.Find("td:nth-child(6) > nobr").Text().Trim(), "dd-MM-yyyyHH:mm:ss", CultureInfo.InvariantCulture); + output("Released on: " + clock.ToShortDateString()); + + // Torrent Details URL + string details = tRow.Find("td:nth-child(2) > a").Attr("href").ToString().TrimStart('/'); + Uri detailsLink = new Uri(SiteLink + details); + output("Details: " + detailsLink.AbsoluteUri); + + // Torrent Comments URL + Uri commentsLink = new Uri(SiteLink + details + "&hit=1&tocomm=1"); + output("Comments Link: " + commentsLink.AbsoluteUri); + + // Torrent Download URL + string download = tRow.Find("td:nth-child(3) > a").Attr("href"); + Uri downloadLink = new Uri(SiteLink + download); + output("Download Link: " + downloadLink.AbsoluteUri); + + // Freeleech + int downloadVolumeFactor = 1; + if (tRow.Find("img[title^=\"FreeLeech\"]").Length >= 1) + { + downloadVolumeFactor = 0; + output("FreeLeech =)"); + } + + // Building release infos + var release = new ReleaseInfo() + { + Category = MapTrackerCatToNewznab(categoryID), + Title = name, + Seeders = seeders, + Peers = seeders + leechers, + Grabs = completed, + MinimumRatio = 1, + MinimumSeedTime = 345600, + PublishDate = clock, + Files = files, + Size = size, + Guid = detailsLink, + Comments = commentsLink, + Link = downloadLink, + UploadVolumeFactor = 1, + DownloadVolumeFactor = downloadVolumeFactor + }; releases.Add(release); } + } catch (Exception ex) { - OnParseError(results, ex); + OnParseError("Error, unable to parse result \n" + ex.StackTrace, ex); } + // Return found releases return releases; } + + /// + /// Build query to process + /// + /// Term to search + /// Torznab Query for categories mapping + /// Search url for provider + /// Page number to request + /// URL to query for parsing and processing results + private string buildQuery(string term, TorznabQuery query, string url, int page = 1) + { + var queryCollection = new NameValueCollection(); + + // Check if we are processing a new page + /*if (page > 1) + { + // Adding page number to query + url += "/" + page.ToString(); + }*/ + + // If search term provided + if (!string.IsNullOrWhiteSpace(term)) + { + // Add search term + queryCollection.Add("search", HttpUtility.UrlEncode(term)); + queryCollection.Add("searchin", "0"); + queryCollection.Add("cat", "0"); + queryCollection.Add("incldead", "0"); + + // Building our query + url += "?" + queryCollection.GetQueryString(); + } + else + { + // Showing all torrents (just for output function) + term = "all"; + } + + output("\nBuilded query for \"" + term + "\"... " + url); + + // Return our search url + return url; + } + + /// + /// Switch Method for Querying + /// + /// URL created by Query Builder + /// Results from query + private async Task queryExec(string request) + { + String results = null; + + // Switch in we are in DEV mode with Hard Drive Cache or not + if (DevMode && CacheMode) + { + // Check Cache before querying and load previous results if available + results = await queryCache(request); + } + else + { + // Querying tracker directly + results = await queryTracker(request); + } + return results; + } + + /// + /// Get Torrents Page from Cache by Query Provided + /// + /// URL created by Query Builder + /// Results from query + private async Task queryCache(string request) + { + String results; + + // Create Directory if not exist + System.IO.Directory.CreateDirectory(Directory); + + // Clean Storage Provider Directory from outdated cached queries + cleanCacheStorage(); + + // File Name + string fileName = StringUtil.HashSHA1(request) + ".json"; + + // Create fingerprint for request + string file = Path.Combine(Directory, fileName); + + // Checking modes states + if (File.Exists(file)) + { + // File exist... loading it right now ! + output("Loading results from hard drive cache ..." + fileName); + try + { + using (StreamReader fileReader = File.OpenText(file)) + { + JsonSerializer serializer = new JsonSerializer(); + results = (String)serializer.Deserialize(fileReader, typeof(String)); + } + } + catch (Exception e) + { + output("Error loading cached results ! " + e.Message, "error"); + results = null; + } + } + else + { + // No cached file found, querying tracker directly + results = await queryTracker(request); + + // Cached file didn't exist for our query, writing it right now ! + output("Writing results to hard drive cache ..." + fileName); + using (StreamWriter fileWriter = File.CreateText(file)) + { + JsonSerializer serializer = new JsonSerializer(); + serializer.Serialize(fileWriter, results); + } + output("Writed to " + Directory + fileName); + } + return results; + } + + /// + /// Get Torrents Page from Tracker by Query Provided + /// + /// URL created by Query Builder + /// Results from query + private async Task queryTracker(string request) + { + WebClientStringResult results = null; + + // Cache mode not enabled or cached file didn't exist for our query + output("\nQuerying tracker for results...."); + + // Request our first page + latencyNow(); + results = await RequestStringWithCookiesAndRetry(request, null, null, emulatedBrowserHeaders); + + // Return results from tracker + return results.Content; + } + + /// + /// Clean Hard Drive Cache Storage + /// + /// Force Provider Folder deletion + private void cleanCacheStorage(bool force = false) + { + // Check cleaning method + if (force) + { + // Deleting Provider Storage folder and all files recursively + output("\nDeleting Provider Storage folder and all files recursively ..."); + + // Check if directory exist + if (System.IO.Directory.Exists(Directory)) + { + // Delete storage directory of provider + System.IO.Directory.Delete(Directory, true); + output("-> Storage folder deleted successfully."); + } + else + { + // No directory, so nothing to do + output("-> No Storage folder found for this provider !"); + } + } + else + { + var i = 0; + // Check if there is file older than ... and delete them + output("\nCleaning Provider Storage folder... in progress."); + System.IO.Directory.GetFiles(Directory) + .Select(f => new FileInfo(f)) + .Where(f => f.LastAccessTime < DateTime.Now.AddMilliseconds(-Convert.ToInt32(ConfigData.HardDriveCacheKeepTime.Value))) + .ToList() + .ForEach(f => + { + output("Deleting cached file << " + f.Name + " >> ... done."); + f.Delete(); + i++; + }); + + // Inform on what was cleaned during process + if (i > 0) + { + output("-> Deleted " + i + " cached files during cleaning."); + } + else + { + output("-> Nothing deleted during cleaning."); + } + } + } + + /// + /// Generate a random fake latency to avoid detection on tracker side + /// + private void latencyNow() + { + // Need latency ? + if (Latency) + { + // Generate a random value in our range + var random = new Random(DateTime.Now.Millisecond); + int waiting = random.Next(Convert.ToInt32(ConfigData.LatencyStart.Value), Convert.ToInt32(ConfigData.LatencyEnd.Value)); + output("\nLatency Faker => Sleeping for " + waiting + " ms..."); + + // Sleep now... + System.Threading.Thread.Sleep(waiting); + } + } + + /// + /// Find torrent rows in search pages + /// + /// JQuery Object + private CQ findTorrentRows() + { + // Return all occurencis of torrents found + return fDom["#minions .browse,.sticky"]; + } + + /// + /// Output message for logging or developpment (console) + /// + /// Message to output + /// Level for Logger + private void output(string message, string level = "debug") + { + // Check if we are in dev mode + if (DevMode) + { + // Output message to console + Console.WriteLine(message); + } + else + { + // Send message to logger with level + switch (level) + { + default: + goto case "debug"; + case "debug": + // Only if Debug Level Enabled on Jackett + if (Engine.Logger.IsDebugEnabled) + { + logger.Debug(message); + } + break; + case "info": + logger.Info(message); + break; + case "error": + logger.Error(message); + break; + } + } + } + + /// + /// Validate Config entered by user on Jackett + /// + private void validateConfig() + { + output("\nValidating Settings ... \n"); + + // Check Username Setting + if (string.IsNullOrEmpty(ConfigData.Username.Value)) + { + throw new ExceptionWithConfigData("You must provide a username for this tracker to login !", ConfigData); + } + else + { + output("Validated Setting -- Username (auth) => " + ConfigData.Username.Value.ToString()); + } + + // Check Password Setting + if (string.IsNullOrEmpty(ConfigData.Password.Value)) + { + throw new ExceptionWithConfigData("You must provide a password with your username for this tracker to login !", ConfigData); + } + else + { + output("Validated Setting -- Password (auth) => " + ConfigData.Password.Value.ToString()); + } + + // Check Max Page Setting + /*if (!string.IsNullOrEmpty(ConfigData.Pages.Value)) + { + try + { + output("Validated Setting -- Max Pages => " + Convert.ToInt32(ConfigData.Pages.Value)); + } + catch (Exception) + { + throw new ExceptionWithConfigData("Please enter a numeric maximum number of pages to crawl !", ConfigData); + } + } + else + { + throw new ExceptionWithConfigData("Please enter a maximum number of pages to crawl !", ConfigData); + }*/ + + // Check Latency Setting + if (ConfigData.Latency.Value) + { + output("\nValidated Setting -- Latency Simulation enabled"); + + // Check Latency Start Setting + if (!string.IsNullOrEmpty(ConfigData.LatencyStart.Value)) + { + try + { + output("Validated Setting -- Latency Start => " + Convert.ToInt32(ConfigData.LatencyStart.Value)); + } + catch (Exception) + { + throw new ExceptionWithConfigData("Please enter a numeric latency start in ms !", ConfigData); + } + } + else + { + throw new ExceptionWithConfigData("Latency Simulation enabled, Please enter a start latency !", ConfigData); + } + + // Check Latency End Setting + if (!string.IsNullOrEmpty(ConfigData.LatencyEnd.Value)) + { + try + { + output("Validated Setting -- Latency End => " + Convert.ToInt32(ConfigData.LatencyEnd.Value)); + } + catch (Exception) + { + throw new ExceptionWithConfigData("Please enter a numeric latency end in ms !", ConfigData); + } + } + else + { + throw new ExceptionWithConfigData("Latency Simulation enabled, Please enter a end latency !", ConfigData); + } + } + + // Check Browser Setting + if (ConfigData.Browser.Value) + { + output("\nValidated Setting -- Browser Simulation enabled"); + + // Check ACCEPT header Setting + if (string.IsNullOrEmpty(ConfigData.HeaderAccept.Value)) + { + throw new ExceptionWithConfigData("Browser Simulation enabled, Please enter an ACCEPT header !", ConfigData); + } + else + { + output("Validated Setting -- ACCEPT (header) => " + ConfigData.HeaderAccept.Value.ToString()); + } + + // Check ACCEPT-LANG header Setting + if (string.IsNullOrEmpty(ConfigData.HeaderAcceptLang.Value)) + { + throw new ExceptionWithConfigData("Browser Simulation enabled, Please enter an ACCEPT-LANG header !", ConfigData); + } + else + { + output("Validated Setting -- ACCEPT-LANG (header) => " + ConfigData.HeaderAcceptLang.Value.ToString()); + } + + // Check USER-AGENT header Setting + if (string.IsNullOrEmpty(ConfigData.HeaderUserAgent.Value)) + { + throw new ExceptionWithConfigData("Browser Simulation enabled, Please enter an USER-AGENT header !", ConfigData); + } + else + { + output("Validated Setting -- USER-AGENT (header) => " + ConfigData.HeaderUserAgent.Value.ToString()); + } + } + + // Check Dev Cache Settings + if (ConfigData.HardDriveCache.Value == true) + { + output("\nValidated Setting -- DEV Hard Drive Cache enabled"); + + // Check if Dev Mode enabled ! + if (!ConfigData.DevMode.Value) + { + throw new ExceptionWithConfigData("Hard Drive is enabled but not in DEV MODE, Please enable DEV MODE !", ConfigData); + } + + // Check Cache Keep Time Setting + if (!string.IsNullOrEmpty(ConfigData.HardDriveCacheKeepTime.Value)) + { + try + { + output("Validated Setting -- Cache Keep Time (ms) => " + Convert.ToInt32(ConfigData.HardDriveCacheKeepTime.Value)); + } + catch (Exception) + { + throw new ExceptionWithConfigData("Please enter a numeric hard drive keep time in ms !", ConfigData); + } + } + else + { + throw new ExceptionWithConfigData("Hard Drive Cache enabled, Please enter a maximum keep time for cache !", ConfigData); + } + } + else + { + // Delete cache if previously existed + cleanCacheStorage(true); + } + } } } - diff --git a/src/Jackett/Jackett.csproj b/src/Jackett/Jackett.csproj index 2dae9bf64..fdb800cd5 100644 --- a/src/Jackett/Jackett.csproj +++ b/src/Jackett/Jackett.csproj @@ -268,7 +268,6 @@ - @@ -382,6 +381,8 @@ + + diff --git a/src/Jackett/Models/IndexerConfig/Bespoke/ConfigurationDataSceneFZ.cs b/src/Jackett/Models/IndexerConfig/Bespoke/ConfigurationDataSceneFZ.cs new file mode 100644 index 000000000..628abcadf --- /dev/null +++ b/src/Jackett/Models/IndexerConfig/Bespoke/ConfigurationDataSceneFZ.cs @@ -0,0 +1,55 @@ +namespace Jackett.Models.IndexerConfig.Bespoke +{ + class ConfigurationDataSceneFZ : ConfigurationData + { + public DisplayItem CredentialsWarning { get; private set; } + public StringItem Username { get; private set; } + public StringItem Password { get; private set; } + public DisplayItem PagesWarning { get; private set; } + //public StringItem Pages { get; private set; } + public BoolItem Dead { get; private set; } + public DisplayItem SecurityWarning { get; private set; } + public BoolItem Latency { get; private set; } + public BoolItem Browser { get; private set; } + public DisplayItem LatencyWarning { get; private set; } + public StringItem LatencyStart { get; private set; } + public StringItem LatencyEnd { get; private set; } + public DisplayItem HeadersWarning { get; private set; } + public StringItem HeaderAccept { get; private set; } + public StringItem HeaderAcceptLang { get; private set; } + public BoolItem HeaderDNT { get; private set; } + public BoolItem HeaderUpgradeInsecure { get; private set; } + public StringItem HeaderUserAgent { get; private set; } + public DisplayItem DevWarning { get; private set; } + public BoolItem DevMode { get; private set; } + public BoolItem HardDriveCache { get; private set; } + public StringItem HardDriveCacheKeepTime { get; private set; } + + public ConfigurationDataSceneFZ() + : base() + { + CredentialsWarning = new DisplayItem("Credentials Configuration (Private Tracker),

  • Username is your account name on this tracker.
  • Password is your password associated to your account name.
") { Name = "Credentials" }; + Username = new StringItem { Name = "Username (Required)", Value = "" }; + Password = new StringItem { Name = "Password (Required)", Value = "" }; + PagesWarning = new DisplayItem("Preferences Configuration (Tweak your search settings),

  • Include Dead let you search including torrents which are marked as Dead.
") { Name = "Preferences" }; + //Pages = new StringItem { Name = "Max Pages to Process (Required)", Value = "4" }; + Dead = new BoolItem() { Name = "Include Dead (Optional)", Value = false }; + SecurityWarning = new DisplayItem("Security Configuration (Read this area carefully !),

  • Latency Simulation will simulate human browsing with Jacket by pausing Jacket for an random time between each request, to fake a real content browsing.
  • Browser Simulation will simulate a real human browser by injecting additionals headers when doing requests to tracker.
") { Name = "Security" }; + Latency = new BoolItem() { Name = "Latency Simulation (Optional)", Value = false }; + Browser = new BoolItem() { Name = "Browser Simulation (Optional)", Value = false }; + LatencyWarning = new DisplayItem("Latency Configuration (Required if latency simulation enabled),

  • By filling this range, Jackett will make a random timed pause between requests to tracker to simulate a real browser.
  • MilliSeconds only
") { Name = "Simulate Latency" }; + LatencyStart = new StringItem { Name = "Minimum Latency (ms)", Value = "1589" }; + LatencyEnd = new StringItem { Name = "Maximum Latency (ms)", Value = "3674" }; + HeadersWarning = new DisplayItem("Browser Headers Configuration (Required if browser simulation enabled),

  • By filling these fields, Jackett will inject headers with your values to simulate a real browser.
  • You can get your browser values here: www.whatismybrowser.com

Note that some headers are not necessary because they are injected automatically by this provider such as Accept_Encoding, Connection, Host or X-Requested-With") { Name = "Injecting headers" }; + HeaderAccept = new StringItem { Name = "Accept", Value = "" }; + HeaderAcceptLang = new StringItem { Name = "Accept-Language", Value = "" }; + HeaderDNT = new BoolItem { Name = "DNT", Value = false }; + HeaderUpgradeInsecure = new BoolItem { Name = "Upgrade-Insecure-Requests", Value = false }; + HeaderUserAgent = new StringItem { Name = "User-Agent", Value = "" }; + DevWarning = new DisplayItem("Development Facility (For Developers ONLY),

  • By enabling development mode, Jackett will bypass his cache and will output debug messages to console instead of his log file.
  • By enabling Hard Drive Cache, This provider will save each query answers from tracker in temp directory, in fact this reduce drastically HTTP requests when building a provider at parsing step for example. So, Jackett will search for a cached query answer on hard drive before executing query on tracker side ! DEV MODE must be enabled to use it !
") { Name = "Development" }; + DevMode = new BoolItem { Name = "Enable DEV MODE (Developers ONLY)", Value = true }; + HardDriveCache = new BoolItem { Name = "Enable HARD DRIVE CACHE (Developers ONLY)", Value = true }; + HardDriveCacheKeepTime = new StringItem { Name = "Keep Cached files for (ms)", Value = "300000" }; + } + } +}