diff --git a/README.md b/README.md index e4cbd5ab0..f955e9e44 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ Developer note: The software implements the [Torznab](https://github.com/Sonarr/ * CinemaZ * DanishBits * Demonoid + * DigitalHive * FileList * Freshon * FunFile diff --git a/src/Jackett/Content/logos/digitalhive.png b/src/Jackett/Content/logos/digitalhive.png new file mode 100644 index 000000000..13d982e2a Binary files /dev/null and b/src/Jackett/Content/logos/digitalhive.png differ diff --git a/src/Jackett/Indexers/DigitalHive.cs b/src/Jackett/Indexers/DigitalHive.cs new file mode 100644 index 000000000..1ee525d95 --- /dev/null +++ b/src/Jackett/Indexers/DigitalHive.cs @@ -0,0 +1,204 @@ +using CsQuery; +using Jackett.Models; +using Jackett.Services; +using Jackett.Utils; +using Jackett.Utils.Clients; +using Newtonsoft.Json.Linq; +using NLog; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Jackett.Models.IndexerConfig; +using System.Collections.Specialized; +using System.Text.RegularExpressions; + +namespace Jackett.Indexers +{ + public class DigitalHive : BaseIndexer, IIndexer + { + private string SearchUrl { get { return SiteLink + "browse.php"; } } + private string LoginUrl { get { return SiteLink + "login.php?returnto=%2F"; } } + private string AjaxLoginUrl { get { return SiteLink + "takelogin.php"; } } + + new ConfigurationDataRecaptchaLogin configData + { + get { return (ConfigurationDataRecaptchaLogin)base.configData; } + set { base.configData = value; } + } + + public DigitalHive(IIndexerManagerService i, Logger l, IWebClient w, IProtectionService ps) + : base(name: "DigitalHive", + description: "DigitalHive is one of the oldest general trackers", + link: "https://www.digitalhive.org/", + caps: new TorznabCapabilities(), + manager: i, + client: w, + logger: l, + p: ps, + configData: new ConfigurationDataRecaptchaLogin()) + { + AddCategoryMapping(0, TorznabCatType.Other); + AddCategoryMapping(48, TorznabCatType.Other); // 0Day + AddCategoryMapping(56, TorznabCatType.XXXImageset); // 0Day-Imagesets + AddCategoryMapping(6, TorznabCatType.Audio); // 0Day-Music + AddCategoryMapping(51, TorznabCatType.XXX); // 0Day-XXX + AddCategoryMapping(2, TorznabCatType.TVAnime); // Anime + AddCategoryMapping(59, TorznabCatType.MoviesBluRay); // BluRay + AddCategoryMapping(40, TorznabCatType.TVDocumentary); // Documentary + AddCategoryMapping(20, TorznabCatType.MoviesDVD); // DVD-R + AddCategoryMapping(25, TorznabCatType.BooksEbook); // Ebooks + AddCategoryMapping(38, TorznabCatType.PCPhoneIOS); // HandHeld + AddCategoryMapping(38, TorznabCatType.PCPhoneAndroid); // HandHeld + AddCategoryMapping(38, TorznabCatType.PCPhoneOther); // HandHeld + AddCategoryMapping(37, TorznabCatType.Other); // Kids Stuff + AddCategoryMapping(23, TorznabCatType.PC); // Linux + AddCategoryMapping(24, TorznabCatType.PCMac); // Mac + AddCategoryMapping(22, TorznabCatType.OtherMisc); // Misc + AddCategoryMapping(35, TorznabCatType.MoviesOther); // Movie Pack + AddCategoryMapping(36, TorznabCatType.MoviesHD); // Movie-HD + AddCategoryMapping(19, TorznabCatType.MoviesSD); // Movie-SD + AddCategoryMapping(50, TorznabCatType.Audio); // Music + AddCategoryMapping(53, TorznabCatType.AudioLossless); // Music-FLAC + AddCategoryMapping(49, TorznabCatType.AudioVideo); // MVID + AddCategoryMapping(1, TorznabCatType.PC); // PC Apps + AddCategoryMapping(4, TorznabCatType.PCGames); // PC Games + AddCategoryMapping(17, TorznabCatType.ConsolePS3); // Playstation + AddCategoryMapping(17, TorznabCatType.ConsolePS4); // Playstation + AddCategoryMapping(17, TorznabCatType.ConsolePSVita); // Playstation + AddCategoryMapping(17, TorznabCatType.ConsolePSP); // Playstation + AddCategoryMapping(28, TorznabCatType.ConsolePSP); // PSP + AddCategoryMapping(34, TorznabCatType.TVOTHER); // TV Pack + AddCategoryMapping(32, TorznabCatType.TVHD); // TV-HD + AddCategoryMapping(55, TorznabCatType.TVOTHER); // TV-HDRip + AddCategoryMapping(7, TorznabCatType.TVSD); // TV-SD + AddCategoryMapping(57, TorznabCatType.TVOTHER); // TV-SDRip + AddCategoryMapping(33, TorznabCatType.ConsoleWii); // WII + AddCategoryMapping(33, TorznabCatType.ConsoleWiiU); // WII + AddCategoryMapping(45, TorznabCatType.ConsoleXbox); // XBox + AddCategoryMapping(45, TorznabCatType.ConsoleXbox360); // XBox + AddCategoryMapping(45, TorznabCatType.ConsoleXBOX360DLC); // XBox + AddCategoryMapping(45, TorznabCatType.ConsoleXboxOne); // XBox + AddCategoryMapping(9, TorznabCatType.XXX); // XXX + AddCategoryMapping(52, TorznabCatType.XXXOther); // XXX-ISO + } + + public override async Task GetConfigurationForSetup() + { + var loginPage = await RequestStringWithCookies(LoginUrl, configData.CookieHeader.Value); + CQ cq = loginPage.Content; + string recaptchaSiteKey = cq.Find(".g-recaptcha").Attr("data-sitekey"); + var result = new ConfigurationDataRecaptchaLogin(); + result.CookieHeader.Value = loginPage.Cookies; + result.Captcha.SiteKey = recaptchaSiteKey; + return result; + } + + public async Task ApplyConfiguration(JToken configJson) + { + configData.LoadValuesFromJson(configJson); + var pairs = new Dictionary { + { "returnto" , "/" }, + { "username", configData.Username.Value }, + { "password", configData.Password.Value }, + { "g-recaptcha-response", configData.Captcha.Value } + }; + + if (!string.IsNullOrWhiteSpace(configData.Captcha.Cookie)) + { + // Cookie was manually supplied + CookieHeader = configData.Captcha.Cookie; + try + { + var results = await PerformQuery(new TorznabQuery()); + if (!results.Any()) + { + throw new Exception("Your cookie did not work"); + } + + SaveConfig(); + IsConfigured = true; + return IndexerConfigurationStatus.Completed; + } + catch (Exception e) + { + IsConfigured = false; + throw new Exception("Your cookie did not work: " + e.Message); + } + } + + var result = await RequestLoginAndFollowRedirect(AjaxLoginUrl, pairs, configData.CookieHeader.Value, true, SiteLink, LoginUrl); + + await ConfigureIfOK(result.Cookies, result.Content.Contains("logout.php"), () => + { + var errorMessage = result.Content; + throw new ExceptionWithConfigData(errorMessage, configData); + }); + + return IndexerConfigurationStatus.RequiresTesting; + } + + public async Task> PerformQuery(TorznabQuery query) + { + List releases = new List(); + + var queryCollection = new NameValueCollection(); + var searchString = query.GetQueryString(); + var searchUrl = SearchUrl; + + foreach (var cat in MapTorznabCapsToTrackers(query)) + { + queryCollection.Add("c" + cat, "1"); + } + + if (!string.IsNullOrWhiteSpace(searchString)) + { + queryCollection.Add("search", searchString); + } + + queryCollection.Add("blah", "0"); + + var results = await RequestStringWithCookiesAndRetry(searchUrl + "?" + queryCollection.GetQueryString()); + try + { + releases.AddRange(contentToReleaseInfos(results.Content)); + } + catch (Exception ex) + { + OnParseError(results.Content, ex); + } + + return releases; + } + + private IEnumerable contentToReleaseInfos(CQ dom) + { + List releases = new List(); + + // Doesn't handle pagination yet... + var rows = dom["div.panel-body > table.table > tbody > tr"]; + foreach (var row in rows) + { + var release = new ReleaseInfo(); + release.MinimumRatio = 1; + release.MinimumSeedTime = 259200; + + var qRow = row.Cq(); + release.Title = qRow.Find("td:nth-child(2) > a").First().Text().Trim(); + release.Description = release.Title; + release.Guid = new Uri(SiteLink + qRow.Find("td:nth-child(2) > a").First().Attr("href")); + release.Comments = release.Guid; + release.Link = new Uri(SiteLink + qRow.Find("td:nth-child(3) > a").First().Attr("href")); + var pubDate = qRow.Find("td:nth-child(2) > span").First().Text().Trim().Replace("Added: ", ""); + release.PublishDate = DateTime.Parse(pubDate).ToLocalTime(); + release.Category = MapTrackerCatToNewznab(qRow.Find("td:nth-child(1) > a").First().Attr("href").Split('=')[1]); + release.Size = ReleaseInfo.GetBytes(qRow.Find("td:nth-child(7)").First().Text()); + release.Seeders = ParseUtil.CoerceInt(qRow.Find("td:nth-child(9)").First().Text()); + release.Peers = ParseUtil.CoerceInt(qRow.Find("td:nth-child(10)").First().Text()) + release.Seeders; + releases.Add(release); + } + + return releases; + } + } +} \ No newline at end of file diff --git a/src/Jackett/Jackett.csproj b/src/Jackett/Jackett.csproj index d3283b07b..5aa81ad43 100644 --- a/src/Jackett/Jackett.csproj +++ b/src/Jackett/Jackett.csproj @@ -50,8 +50,8 @@ 4 - - ..\packages\CloudFlareUtilities.0.3.1-alpha\lib\portable45-net45+win8+wpa81\CloudFlareUtilities.dll + + ..\packages\CloudFlareUtilities.0.3.2-alpha\lib\portable45-net45+win8+wpa81\CloudFlareUtilities.dll True diff --git a/src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs b/src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs index da2f38cc8..40ab5b3af 100644 --- a/src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs +++ b/src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs @@ -11,39 +11,17 @@ using System.Net; using System.Net.Http; using System.Text; using System.Threading.Tasks; -using System.Reflection; +using CloudFlareUtilities; namespace Jackett.Utils.Clients { public class UnixLibCurlWebClient : IWebClient { private Logger logger; - private static Boolean CloudFlareUtilitiesInit = false; - private static Assembly CloudFlareUtilities = null; - private static MethodInfo ChallengeSolverSolveMethod = null; - private static MethodInfo ChallengeSolutionGetClearanceQueryMethod = null; public UnixLibCurlWebClient(Logger l) { logger = l; - - // try loading CloudFlareUtilities - if (!CloudFlareUtilitiesInit) - { - try - { - // use reflections to get the internal CloudFlareUtilities methods we need - CloudFlareUtilities = Assembly.LoadFile(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "/CloudFlareUtilities.dll"); - ChallengeSolverSolveMethod = CloudFlareUtilities.GetType("CloudFlareUtilities.ChallengeSolver").GetMethod("Solve"); - ChallengeSolutionGetClearanceQueryMethod = CloudFlareUtilities.GetType("CloudFlareUtilities.ChallengeSolution").GetMethod("get_ClearanceQuery"); - } - catch (Exception e) - { - Engine.Logger.Error(e.ToString()); - Engine.Logger.Error(string.Format("UnixLibCurlWebClient: Error while loading CloudFlareUtilities.dll from {0}", Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location))); - } - } - CloudFlareUtilitiesInit = true; } public async Task GetBytes(WebRequest request) @@ -64,21 +42,20 @@ namespace Jackett.Utils.Clients private string CloudFlareChallengeSolverSolve(string challengePageContent, Uri uri) { - var solution = ChallengeSolverSolveMethod.Invoke(null, new object[] { challengePageContent, uri.Host }); - string clearanceQuery = (string)ChallengeSolutionGetClearanceQueryMethod.Invoke(solution, new object[] { }); - string clearanceUri = uri.Scheme + Uri.SchemeDelimiter + uri.Host + ":" + uri.Port + clearanceQuery; + var solution = ChallengeSolver.Solve(challengePageContent, uri.Host); + string clearanceUri = uri.Scheme + Uri.SchemeDelimiter + uri.Host + ":" + uri.Port + solution.ClearanceQuery; return clearanceUri; - } - + } + public void Init() { try { Engine.Logger.Info("LibCurl init " + Curl.GlobalInit(CurlInitFlag.All).ToString()); - CurlHelper.OnErrorMessage += (msg) => - { - Engine.Logger.Error(msg); - }; + CurlHelper.OnErrorMessage += (msg) => + { + Engine.Logger.Error(msg); + }; } catch (Exception e) { @@ -107,11 +84,6 @@ namespace Jackett.Utils.Clients if (result.Status == HttpStatusCode.ServiceUnavailable && ((request.Cookies != null && request.Cookies.Contains("__cfduid")) || result.Cookies.Contains("__cfduid"))) { logger.Info("UnixLibCurlWebClient: Received a new CloudFlare challenge"); - if(ChallengeSolutionGetClearanceQueryMethod == null) - { - logger.Error("UnixLibCurlWebClient: CloudFlareUtilities not available, can't solve challenge"); - return result; - } // solve the challenge string pageContent = Encoding.UTF8.GetString(result.Content); @@ -156,7 +128,7 @@ namespace Jackett.Utils.Clients logger.Debug("UnixLibCurlWebClient: Posting " + StringUtil.PostDataFromDict(request.PostData)); } - response = await CurlHelper.PostAsync(request.Url, request.PostData, request.Cookies, request.Referer, request.RawBody); + response = await CurlHelper.PostAsync(request.Url, request.PostData, request.Cookies, request.Referer, request.RawBody); } var result = new WebClientByteResult() diff --git a/src/Jackett/packages.config b/src/Jackett/packages.config index 3f38d3cd5..5e9e802c5 100644 --- a/src/Jackett/packages.config +++ b/src/Jackett/packages.config @@ -7,7 +7,7 @@ - +