diff --git a/src/Jackett.Test/Jackett.Test.csproj b/src/Jackett.Test/Jackett.Test.csproj index 8d0bbc459..96632e5d3 100644 --- a/src/Jackett.Test/Jackett.Test.csproj +++ b/src/Jackett.Test/Jackett.Test.csproj @@ -158,6 +158,7 @@ + diff --git a/src/Jackett.Test/Util/ServerUtilTests.cs b/src/Jackett.Test/Util/ServerUtilTests.cs new file mode 100644 index 000000000..c59b942e5 --- /dev/null +++ b/src/Jackett.Test/Util/ServerUtilTests.cs @@ -0,0 +1,50 @@ +using Jackett.Utils.Clients; +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Autofac; +using Jackett.Indexers; +using FluentAssertions; +using Newtonsoft.Json.Linq; +using Jackett; +using Newtonsoft.Json; +using Jackett.Utils; + +namespace JackettTest.Indexers +{ + [TestFixture] + class ServerUtilTests : TestBase + { + [Test] + public void ResureRedirectIsFullyQualified_makes_redicts_fully_qualified() + { + var res = new WebClientByteResult() + { + RedirectingTo = "list?p=1" + }; + + var req = new WebRequest() + { + Url = "http://my.domain.com/page.php" + }; + + // Not fully qualified requiring redirect + ServerUtil.ResureRedirectIsFullyQualified(req, res); + Assert.AreEqual(res.RedirectingTo, "http://my.domain.com/list?p=1"); + + // Fully qualified not needing modified + res.RedirectingTo = "http://a.domain/page.htm"; + ServerUtil.ResureRedirectIsFullyQualified(req, res); + Assert.AreEqual(res.RedirectingTo, "http://a.domain/page.htm"); + + // Relative requiring redirect + req.Url = "http://my.domain.com/dir/page.php"; + res.RedirectingTo = "a/dir/page.html"; + ServerUtil.ResureRedirectIsFullyQualified(req, res); + Assert.AreEqual(res.RedirectingTo, "http://my.domain.com/dir/a/dir/page.html"); + } + } +} diff --git a/src/Jackett/Controllers/APIController.cs b/src/Jackett/Controllers/APIController.cs index 1ba8fac6d..bf4d94270 100644 --- a/src/Jackett/Controllers/APIController.cs +++ b/src/Jackett/Controllers/APIController.cs @@ -85,10 +85,10 @@ namespace Jackett.Controllers { Title = indexer.DisplayName, Description = indexer.DisplayDescription, - Link = indexer.SiteLink, + Link = new Uri(indexer.SiteLink), ImageUrl = new Uri(severUrl + "logos/" + indexer.ID + ".png"), ImageTitle = indexer.DisplayName, - ImageLink = indexer.SiteLink, + ImageLink = new Uri(indexer.SiteLink), ImageDescription = indexer.DisplayName }); diff --git a/src/Jackett/Controllers/AdminController.cs b/src/Jackett/Controllers/AdminController.cs index 4900f951a..77a27545d 100644 --- a/src/Jackett/Controllers/AdminController.cs +++ b/src/Jackett/Controllers/AdminController.cs @@ -1,4 +1,5 @@ using Autofac; +using Jackett.Indexers; using Jackett.Models; using Jackett.Services; using Jackett.Utils; @@ -160,12 +161,13 @@ namespace Jackett.Controllers [HttpPost] public async Task Configure() { - JToken jsonReply = new JObject(); + var jsonReply = new JObject(); + IIndexer indexer = null; try { var postData = await ReadPostDataJson(); string indexerString = (string)postData["indexer"]; - var indexer = indexerService.GetIndexer((string)postData["indexer"]); + indexer = indexerService.GetIndexer((string)postData["indexer"]); jsonReply["name"] = indexer.DisplayName; await indexer.ApplyConfiguration(postData["config"]); await indexerService.TestIndexer((string)postData["indexer"]); @@ -175,6 +177,9 @@ namespace Jackett.Controllers { jsonReply["result"] = "error"; jsonReply["error"] = ex.Message; + var baseIndexer = indexer as BaseIndexer; + if (null != baseIndexer) + baseIndexer.ResetBaseConfig(); if (ex is ExceptionWithConfigData) { jsonReply["config"] = ((ExceptionWithConfigData)ex).ConfigData.ToJson(); diff --git a/src/Jackett/Indexers/AlphaRatio.cs b/src/Jackett/Indexers/AlphaRatio.cs index 5a39584c5..7effde082 100644 --- a/src/Jackett/Indexers/AlphaRatio.cs +++ b/src/Jackett/Indexers/AlphaRatio.cs @@ -19,100 +19,48 @@ namespace Jackett.Indexers { public class AlphaRatio : BaseIndexer, IIndexer { - private readonly string LoginUrl = ""; - private readonly string SearchUrl = ""; - private readonly string DownloadUrl = ""; - private readonly string GuidUrl = ""; - - private IWebClient webclient; - private string cookieHeader = ""; + private string LoginUrl { get { return SiteLink + "login.php"; } } + private string SearchUrl { get { return SiteLink + "ajax.php?action=browse&searchstr="; } } + private string DownloadUrl { get { return SiteLink + "torrents.php?action=download&id="; } } + private string GuidUrl { get { return SiteLink + "torrents.php?torrentid="; } } public AlphaRatio(IIndexerManagerService i, IWebClient w, Logger l) : base(name: "AlphaRatio", description: "Legendary", - link: new Uri("https://alpharatio.cc"), + link: "https://alpharatio.cc/", caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), manager: i, + client: w, logger: l) { - LoginUrl = SiteLink + "login.php"; - SearchUrl = SiteLink + "ajax.php?action=browse&searchstr="; - DownloadUrl = SiteLink + "torrents.php?action=download&id="; - GuidUrl = SiteLink + "torrents.php?torrentid="; webclient = w; } public Task GetConfigurationForSetup() { - var config = new ConfigurationDataBasicLogin(); - return Task.FromResult(config); + return Task.FromResult(new ConfigurationDataBasicLogin()); } public async Task ApplyConfiguration(JToken configJson) { var incomingConfig = new ConfigurationDataBasicLogin(); incomingConfig.LoadValuesFromJson(configJson); - var pairs = new Dictionary { - { "username", incomingConfig.Username.Value }, - { "password", incomingConfig.Password.Value }, - { "login", "Login" }, - { "keeplogged", "1" } - }; + { "username", incomingConfig.Username.Value }, + { "password", incomingConfig.Password.Value }, + { "login", "Login" }, + { "keeplogged", "1" } + }; // Do the login - var response = await webclient.GetString(new Utils.Clients.WebRequest() - { - PostData = pairs, - Referer = LoginUrl, - Type = RequestType.POST, - Url = LoginUrl - }); - - cookieHeader = response.Cookies; - - if (response.Status == HttpStatusCode.Found || response.Status == HttpStatusCode.Redirect || response.Status == HttpStatusCode.RedirectMethod) - { - response = await webclient.GetString(new Utils.Clients.WebRequest() - { - Url = SiteLink.ToString(), - Referer = LoginUrl.ToString(), - Cookies = cookieHeader - }); - } - - if (!response.Content.Contains("logout.php?")) - { - CQ dom = response.Content; - dom["#loginform > table"].Remove(); - var errorMessage = dom["#loginform"].Text().Trim().Replace("\n\t", " "); - throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)incomingConfig); - } - else - { - var configSaveData = new JObject(); - configSaveData["cookie_header"] = cookieHeader; - SaveConfig(configSaveData); - IsConfigured = true; - } - } - - HttpRequestMessage CreateHttpRequest(Uri uri) - { - var message = new HttpRequestMessage(); - message.Method = HttpMethod.Post; - message.RequestUri = uri; - message.Headers.UserAgent.ParseAdd(BrowserUtil.ChromeUserAgent); - return message; - } - - public void LoadFromSavedConfiguration(JToken jsonConfig) - { - cookieHeader = (string)jsonConfig["cookie_header"]; - if (!string.IsNullOrEmpty(cookieHeader)) - { - IsConfigured = true; - } + var response = await RequestLoginAndFollowRedirect(LoginUrl, pairs, string.Empty, true, SiteLink); + ConfigureIfOK(response.Cookies, response.Content!=null && response.Content.Contains("logout.php?"), () => + { + CQ dom = response.Content; + dom["#loginform > table"].Remove(); + var errorMessage = dom["#loginform"].Text().Trim().Replace("\n\t", " "); + throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)incomingConfig); + }); } void FillReleaseInfoFromJson(ReleaseInfo release, JObject r) @@ -128,22 +76,14 @@ namespace Jackett.Indexers public async Task PerformQuery(TorznabQuery query) { - List releases = new List(); + var releases = new List(); var searchString = query.SanitizedSearchTerm + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = SearchUrl + HttpUtility.UrlEncode(searchString); - - var results = await webclient.GetString(new Utils.Clients.WebRequest() - { - Cookies = cookieHeader, - Type = RequestType.GET, - Url = episodeSearchUrl - }); - + var response = await RequestStringWithCookies(episodeSearchUrl); try { - - var json = JObject.Parse(results.Content); + var json = JObject.Parse(response.Content); foreach (JObject r in json["response"]["results"]) { DateTime pubDate = DateTime.MinValue; @@ -179,7 +119,7 @@ namespace Jackett.Indexers } catch (Exception ex) { - OnParseError(results.Content, ex); + OnParseError(response.Content, ex); } return releases.ToArray(); @@ -191,17 +131,5 @@ namespace Jackett.Indexers long unixTimeStampInTicks = (long)(unixTime * TimeSpan.TicksPerSecond); return new DateTime(unixStart.Ticks + unixTimeStampInTicks); } - - public async Task Download(Uri link) - { - var response = await webclient.GetBytes(new Utils.Clients.WebRequest() - { - Url = link.ToString(), - Cookies = cookieHeader - }); - - return response.Content; - } - } } diff --git a/src/Jackett/Indexers/AnimeBytes.cs b/src/Jackett/Indexers/AnimeBytes.cs index 613888f98..1f7f97f86 100644 --- a/src/Jackett/Indexers/AnimeBytes.cs +++ b/src/Jackett/Indexers/AnimeBytes.cs @@ -1,5 +1,6 @@ using CsQuery; using Jackett.Models; +using Jackett.Models.IndexerConfig; using Jackett.Services; using Jackett.Utils; using Jackett.Utils.Clients; @@ -22,50 +23,24 @@ namespace Jackett.Indexers { public class AnimeBytes : BaseIndexer, IIndexer { - class ConfigurationDataBasicLoginAnimeBytes : ConfigurationDataBasicLogin - { - public BoolItem IncludeRaw { get; private set; } - public DisplayItem DateWarning { get; private set; } - - public ConfigurationDataBasicLoginAnimeBytes() - : base() - { - IncludeRaw = new BoolItem() { Name = "IncludeRaw", Value = false }; - DateWarning = new DisplayItem("This tracker does not supply upload dates so they are based off year of release.") { Name = "DateWarning" }; - } - - public override Item[] GetItems() - { - return new Item[] { Username, Password, IncludeRaw, DateWarning }; - } - } - - private readonly string LoginUrl = ""; - private readonly string SearchUrl = ""; + private string LoginUrl { get { return SiteLink + "user/login"; } } + private string SearchUrl { get { return SiteLink + "torrents.php?filter_cat[1]=1"; } } public bool AllowRaws { get; private set; } - private IWebClient webclient; - private string cookieHeader = ""; - public AnimeBytes(IIndexerManagerService i, IWebClient client, Logger l) : base(name: "AnimeBytes", + link: "https://animebytes.tv/", description: "The web's best Chinese cartoons", - link: new Uri("https://animebytes.tv"), - caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), manager: i, + client: client, + caps: new TorznabCapabilities(TorznabCategory.Anime), logger: l) { - TorznabCaps.Categories.Clear(); - TorznabCaps.Categories.Add(new TorznabCategory { ID = "5070", Name = "TV/Anime" }); - LoginUrl = SiteLink + "user/login"; - SearchUrl = SiteLink + "torrents.php?filter_cat[1]=1"; - webclient = client; } public Task GetConfigurationForSetup() { - var config = new ConfigurationDataBasicLoginAnimeBytes(); - return Task.FromResult(config); + return Task.FromResult(new ConfigurationDataBasicLoginAnimeBytes()); } public async Task ApplyConfiguration(JToken configJson) @@ -73,7 +48,6 @@ namespace Jackett.Indexers var config = new ConfigurationDataBasicLoginAnimeBytes(); config.LoadValuesFromJson(configJson); - // Get the login form as we need the CSRF Token var loginPage = await webclient.GetString(new Utils.Clients.WebRequest() { @@ -85,46 +59,36 @@ namespace Jackett.Indexers // Build login form var pairs = new Dictionary { - { "csrf_token", csrfToken.Attr("value") }, - { "username", config.Username.Value }, - { "password", config.Password.Value }, - { "keeplogged_sent", "true" }, - { "keeplogged", "on" }, - { "login", "Log In!" } + { "csrf_token", csrfToken.Attr("value") }, + { "username", config.Username.Value }, + { "password", config.Password.Value }, + { "keeplogged_sent", "true" }, + { "keeplogged", "on" }, + { "login", "Log In!" } }; - var content = new FormUrlEncodedContent(pairs); - // Do the login - var response = await webclient.GetString(new Utils.Clients.WebRequest() + var request = new Utils.Clients.WebRequest() { Cookies = loginPage.Cookies, - PostData = pairs, - Referer = LoginUrl, - Type = RequestType.POST, - Url = LoginUrl - }); + PostData = pairs, + Referer = LoginUrl, + Type = RequestType.POST, + Url = LoginUrl + }; + var response = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, null); // Follow the redirect - if (response.Status == HttpStatusCode.RedirectMethod) - { - cookieHeader = response.Cookies; - response = await webclient.GetString(new Utils.Clients.WebRequest() - { - Url = SearchUrl, - PostData = pairs, - Referer = SiteLink.ToString(), - Cookies = cookieHeader - }); - } + await FollowIfRedirect(request, response, SearchUrl); - if (!response.Content.Contains("/user/logout")) + if (!(response.Content != null && response.Content.Contains("/user/logout"))) { // Their login page appears to be broken and just gives a 500 error. throw new ExceptionWithConfigData("Failed to login, 6 failed attempts will get you banned for 6 hours.", (ConfigurationData)config); } else { + cookieHeader = response.Cookies; AllowRaws = config.IncludeRaw.Value; var configSaveData = new JObject(); configSaveData["cookies"] = cookieHeader; @@ -134,7 +98,7 @@ namespace Jackett.Indexers } } - public void LoadFromSavedConfiguration(JToken jsonConfig) + public override void LoadFromSavedConfiguration(JToken jsonConfig) { // The old config used an array - just fail to load it if (!(jsonConfig["cookies"] is JArray)) @@ -145,23 +109,6 @@ namespace Jackett.Indexers } } - - private string Hash(string input) - { - // Use input string to calculate MD5 hash - MD5 md5 = System.Security.Cryptography.MD5.Create(); - byte[] inputBytes = System.Text.Encoding.ASCII.GetBytes(input); - byte[] hashBytes = md5.ComputeHash(inputBytes); - - // Convert the byte array to hexadecimal string - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < hashBytes.Length; i++) - { - sb.Append(hashBytes[i].ToString("X2")); - } - return sb.ToString(); - } - public async Task PerformQuery(TorznabQuery query) { // The result list @@ -207,12 +154,7 @@ namespace Jackett.Indexers } // Get the content from the tracker - var response = await webclient.GetString(new Utils.Clients.WebRequest() - { - Cookies = cookieHeader, - Url = queryUrl, - Type = RequestType.GET - }); + var response = await RequestStringWithCookies(queryUrl); CQ dom = response.Content; // Parse @@ -311,7 +253,7 @@ namespace Jackett.Indexers var infoLink = links.Get(1); release.Comments = new Uri(SiteLink + "/" + infoLink.Attributes.GetAttribute("href")); - release.Guid = new Uri(SiteLink + "/" + infoLink.Attributes.GetAttribute("href") + "&nh=" + Hash(title)); // Sonarr should dedupe on this url - allow a url per name. + release.Guid = new Uri(SiteLink + "/" + infoLink.Attributes.GetAttribute("href") + "&nh=" + StringUtil.Hash(title)); // Sonarr should dedupe on this url - allow a url per name. release.Link = new Uri(SiteLink + "/" + downloadLink.Attributes.GetAttribute("href")); // We dont actually have a release name >.> so try to create one @@ -382,16 +324,5 @@ namespace Jackett.Indexers return releases.Select(s => (ReleaseInfo)s.Clone()).ToArray(); } - - public async Task Download(Uri link) - { - var response = await webclient.GetBytes(new Utils.Clients.WebRequest() - { - Url = link.ToString(), - Cookies = cookieHeader - }); - - return response.Content; - } } } diff --git a/src/Jackett/Indexers/BB.cs b/src/Jackett/Indexers/BB.cs index 9e7c39251..edc9e3ca0 100644 --- a/src/Jackett/Indexers/BB.cs +++ b/src/Jackett/Indexers/BB.cs @@ -2,6 +2,7 @@ using Jackett.Models; using Jackett.Services; using Jackett.Utils; +using Jackett.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using System; @@ -19,61 +20,41 @@ namespace Jackett.Indexers public class BB : BaseIndexer, IIndexer { - private readonly string BaseUrl = ""; - private readonly string LoginUrl = ""; - private readonly string SearchUrl = ""; + private string BaseUrl { get { return StringUtil.FromBase64("aHR0cHM6Ly9iYWNvbmJpdHMub3JnLw=="); } } + private string LoginUrl { get { return SiteLink + "login.php"; } } + private string SearchUrl { get { return SiteLink + "torrents.php?searchstr={0}&searchtags=&tags_type=0&order_by=s3&order_way=desc&disablegrouping=1&filter_cat%5B10%5D=1"; } } - CookieContainer cookies; - HttpClientHandler handler; - HttpClient client; - - public BB(IIndexerManagerService i, Logger l) + public BB(IIndexerManagerService i, Logger l, IWebClient w) : base(name: "bB", description: "bB", - link: new Uri("http://www.reddit.com/r/baconbits"), + link: "http://www.reddit.com/r/baconbits/", caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), manager: i, + client: w, logger: l) { - - BaseUrl = StringUtil.FromBase64("aHR0cHM6Ly9iYWNvbmJpdHMub3Jn"); - LoginUrl = BaseUrl + "/login.php"; - SearchUrl = BaseUrl + "/torrents.php?searchstr={0}&searchtags=&tags_type=0&order_by=s3&order_way=desc&disablegrouping=1&filter_cat%5B10%5D=1"; - cookies = new CookieContainer(); - handler = new HttpClientHandler - { - CookieContainer = cookies, - AllowAutoRedirect = true, - UseCookies = true, - }; - client = new HttpClient(handler); } public Task GetConfigurationForSetup() { - var config = new ConfigurationDataBasicLogin(); - return Task.FromResult(config); + return Task.FromResult(new ConfigurationDataBasicLogin()); } public async Task ApplyConfiguration(JToken configJson) { - var config = new ConfigurationDataBasicLogin(); - config.LoadValuesFromJson(configJson); - + var incomingConfig = new ConfigurationDataBasicLogin(); + incomingConfig.LoadValuesFromJson(configJson); var pairs = new Dictionary { - { "username", config.Username.Value }, - { "password", config.Password.Value }, + { "username", incomingConfig.Username.Value }, + { "password", incomingConfig.Password.Value }, { "keeplogged", "1" }, { "login", "Log In!" } - }; + }; - var content = new FormUrlEncodedContent(pairs); - var response = await client.PostAsync(LoginUrl, content); - var responseContent = await response.Content.ReadAsStringAsync(); - - if (!responseContent.Contains("logout.php")) + var response = await RequestLoginAndFollowRedirect(LoginUrl, pairs, null, true, null, SiteLink); + ConfigureIfOK(response.Cookies, response.Content != null && response.Content.Contains("logout.php"), () => { - CQ dom = responseContent; + CQ dom = response.Content; var messageEl = dom["#loginform"]; var messages = new List(); for (var i = 0; i < 13; i++) @@ -82,23 +63,9 @@ namespace Jackett.Indexers messages.Add(child.Cq().Text().Trim()); } var message = string.Join(" ", messages); - throw new ExceptionWithConfigData(message, (ConfigurationData)config); - } - else - { + throw new ExceptionWithConfigData(message, (ConfigurationData)incomingConfig); - var configSaveData = new JObject(); - cookies.DumpToJson(BaseUrl, configSaveData); - SaveConfig(configSaveData); - IsConfigured = true; - } - - } - - public void LoadFromSavedConfiguration(JToken jsonConfig) - { - cookies.FillFromJson(new Uri(BaseUrl), jsonConfig, logger); - IsConfigured = true; + }); } public async Task PerformQuery(TorznabQuery query) @@ -107,10 +74,11 @@ namespace Jackett.Indexers var searchString = query.SanitizedSearchTerm + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = string.Format(SearchUrl, HttpUtility.UrlEncode(searchString)); - var results = await client.GetStringAsync(episodeSearchUrl); + var results = await RequestStringWithCookies(episodeSearchUrl); + try { - CQ dom = results; + CQ dom = results.Content; var rows = dom["#torrent_table > tbody > tr.torrent"]; foreach (var row in rows) { @@ -143,15 +111,9 @@ namespace Jackett.Indexers } catch (Exception ex) { - OnParseError(results, ex); + OnParseError(results.Content, ex); } return releases.ToArray(); } - - public Task Download(Uri link) - { - return client.GetByteArrayAsync(link); - } - } } diff --git a/src/Jackett/Indexers/BakaBT.cs b/src/Jackett/Indexers/BakaBT.cs index 647c6f878..46ccee3f9 100644 --- a/src/Jackett/Indexers/BakaBT.cs +++ b/src/Jackett/Indexers/BakaBT.cs @@ -18,31 +18,23 @@ namespace Jackett.Indexers { public class BakaBT : BaseIndexer, IIndexer { - public readonly string SearchUrl = ""; - public readonly string LoginUrl = ""; - private string cookieHeader = ""; - private IWebClient webclient; + public string SearchUrl { get { return SiteLink + "browse.php?only=0&hentai=1&incomplete=1&lossless=1&hd=1&multiaudio=1&bonus=1&c1=1&reorder=1&q="; } } + public string LoginUrl { get { return SiteLink + "login.php"; } } public BakaBT(IIndexerManagerService i, IWebClient wc, Logger l) : base(name: "BakaBT", description: "Anime Community", - link: new Uri("http://bakabt.me/"), - caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), + link: "http://bakabt.me/", + caps: new TorznabCapabilities(TorznabCategory.Anime), manager: i, + client: wc, logger: l) { - TorznabCaps.Categories.Clear(); - TorznabCaps.Categories.Add(new TorznabCategory { ID = "5070", Name = "TV/Anime" }); - - SearchUrl = SiteLink + "browse.php?only=0&hentai=1&incomplete=1&lossless=1&hd=1&multiaudio=1&bonus=1&c1=1&reorder=1&q="; - LoginUrl = SiteLink + "login.php"; - webclient = wc; } public Task GetConfigurationForSetup() { - var config = new ConfigurationDataBasicLogin(); - return Task.FromResult((ConfigurationData)config); + return Task.FromResult(new ConfigurationDataBasicLogin()); } public async Task ApplyConfiguration(JToken configJson) @@ -62,47 +54,15 @@ namespace Jackett.Indexers { "returnto", "/index.php" } }; - var response = await webclient.GetString(new Utils.Clients.WebRequest() - { - Url = LoginUrl, - PostData = pairs, - Referer = SiteLink.ToString(), - Type = RequestType.POST, - Cookies = loginForm.Cookies - }); - - cookieHeader = response.Cookies; - if (response.Status == HttpStatusCode.Found) - { - response = await webclient.GetString(new Utils.Clients.WebRequest() - { - Url = SearchUrl, - Cookies = cookieHeader - }); - } - + var response = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginForm.Cookies, true, null, SiteLink); var responseContent = response.Content; - - if (!responseContent.Contains("Logout")) - { - CQ dom = responseContent; - var messageEl = dom[".error"].First(); - var errorMessage = messageEl.Text().Trim(); - throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)config); - } - else - { - var configSaveData = new JObject(); - configSaveData["cookies"] = cookieHeader; - SaveConfig(configSaveData); - IsConfigured = true; - } - } - - public void LoadFromSavedConfiguration(JToken jsonConfig) - { - cookieHeader = (string)jsonConfig["cookies"]; - IsConfigured = true; + ConfigureIfOK(response.Cookies, responseContent.Contains("Logout"), () => + { + CQ dom = responseContent; + var messageEl = dom[".error"].First(); + var errorMessage = messageEl.Text().Trim(); + throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)config); + }); } public async Task PerformQuery(TorznabQuery query) @@ -119,12 +79,7 @@ namespace Jackett.Indexers var releases = new List(); var searchString = query.SanitizedSearchTerm; var episodeSearchUrl = SearchUrl + HttpUtility.UrlEncode(searchString); - - var response = await webclient.GetString(new Utils.Clients.WebRequest() - { - Url = episodeSearchUrl, - Cookies = cookieHeader - }); + var response = await RequestStringWithCookies(episodeSearchUrl); try { @@ -218,14 +173,9 @@ namespace Jackett.Indexers return releases.ToArray(); } - public async Task Download(Uri link) + public override async Task Download(Uri link) { - var downloadPage = await webclient.GetString(new Utils.Clients.WebRequest() - { - Url = link.ToString(), - Cookies = cookieHeader - }); - + var downloadPage = await RequestStringWithCookies(link.ToString()); CQ dom = downloadPage.Content; var downloadLink = dom.Find(".download_link").First().Attr("href"); @@ -234,14 +184,7 @@ namespace Jackett.Indexers throw new Exception("Unable to find download link."); } - downloadLink = SiteLink + downloadLink; - - var response = await webclient.GetBytes(new Utils.Clients.WebRequest() - { - Url = downloadLink, - Cookies = cookieHeader - }); - + var response = await RequestBytesWithCookies(SiteLink + downloadLink); return response.Content; } } diff --git a/src/Jackett/Indexers/BaseIndexer.cs b/src/Jackett/Indexers/BaseIndexer.cs index 9151e36dd..9128109ab 100644 --- a/src/Jackett/Indexers/BaseIndexer.cs +++ b/src/Jackett/Indexers/BaseIndexer.cs @@ -8,39 +8,53 @@ using Newtonsoft.Json.Linq; using NLog; using Jackett.Services; using Jackett.Utils; +using Jackett.Utils.Clients; +using AutoMapper; namespace Jackett.Indexers { public abstract class BaseIndexer { + public string SiteLink { get; private set; } public string DisplayDescription { get; private set; } public string DisplayName { get; private set; } public string ID { get { return GetIndexerID(GetType()); } } public bool IsConfigured { get; protected set; } - public Uri SiteLink { get; private set; } - public TorznabCapabilities TorznabCaps { get; private set; } - protected Logger logger; protected IIndexerManagerService indexerService; - protected static List cache = new List(); protected static readonly TimeSpan cacheTime = new TimeSpan(0, 9, 0); + protected IWebClient webclient; + protected string cookieHeader = ""; + + public BaseIndexer(string name, string link, string description, IIndexerManagerService manager, IWebClient client, Logger logger, TorznabCapabilities caps = null) + { + if (!link.EndsWith("/")) + throw new Exception("Site link must end with a slash."); + + DisplayName = name; + DisplayDescription = description; + SiteLink = link; + this.logger = logger; + indexerService = manager; + webclient = client; + + if (caps == null) + caps = TorznabCapsUtil.CreateDefaultTorznabTVCaps(); + TorznabCaps = caps; + } public static string GetIndexerID(Type type) { return StringUtil.StripNonAlphaNumeric(type.Name.ToLowerInvariant()); } - public BaseIndexer(string name, string description, Uri link, TorznabCapabilities caps, IIndexerManagerService manager,Logger logger) + public void ResetBaseConfig() { - DisplayName = name; - DisplayDescription = description; - SiteLink = link; - TorznabCaps = caps; - this.logger = logger; - indexerService = manager; + cookieHeader = string.Empty; + IsConfigured = false; } protected void SaveConfig(JToken config) @@ -64,5 +78,158 @@ namespace Jackett.Indexers cache.Remove(expired); } } + + protected async Task FollowIfRedirect(WebRequest request, WebClientStringResult incomingResponse, string overrideRedirectUrl = null, string overrideCookies = null) + { + if (incomingResponse.Status == System.Net.HttpStatusCode.Redirect || + incomingResponse.Status == System.Net.HttpStatusCode.RedirectKeepVerb || + incomingResponse.Status == System.Net.HttpStatusCode.RedirectMethod || + incomingResponse.Status == System.Net.HttpStatusCode.Found) + { + // Do redirect + var redirectedResponse = await webclient.GetString(new WebRequest() + { + Url = overrideRedirectUrl??incomingResponse.RedirectingTo, + Referer = request.Url, + Cookies = overrideCookies??cookieHeader + }); + Mapper.Map(redirectedResponse, incomingResponse); + } + } + + protected async void FollowIfRedirect(WebRequest request, WebClientByteResult incomingResponse, string overrideRedirectUrl) + { + if (incomingResponse.Status == System.Net.HttpStatusCode.Redirect || + incomingResponse.Status == System.Net.HttpStatusCode.RedirectKeepVerb || + incomingResponse.Status == System.Net.HttpStatusCode.RedirectMethod || + incomingResponse.Status == System.Net.HttpStatusCode.Found) + { + // Do redirect + var redirectedResponse = await webclient.GetBytes(new WebRequest() + { + Url = overrideRedirectUrl??incomingResponse.RedirectingTo, + Referer = request.Url, + Cookies = cookieHeader + }); + Mapper.Map(redirectedResponse, incomingResponse); + } + } + + protected void LoadCookieHeaderAndConfigure(JToken jsonConfig) + { + cookieHeader = (string)jsonConfig["cookie_header"]; + if (!string.IsNullOrEmpty(cookieHeader)) + { + IsConfigured = true; + } + else + { + // Legacy cookie key + cookieHeader = (string)jsonConfig["cookies"]; + if (!string.IsNullOrEmpty(cookieHeader)) + { + IsConfigured = true; + } + } + } + + protected void SaveCookieHeaderAndConfigure() + { + var configSaveData = new JObject(); + configSaveData["cookie_header"] = cookieHeader; + SaveConfig(configSaveData); + IsConfigured = !string.IsNullOrEmpty(cookieHeader); + } + + public virtual void LoadFromSavedConfiguration(JToken jsonConfig) + { + LoadCookieHeaderAndConfigure(jsonConfig); + } + + public async virtual Task Download(Uri link) + { + var response = await webclient.GetBytes(new Utils.Clients.WebRequest() + { + Url = link.ToString(), + Cookies = cookieHeader + }); + + return response.Content; + } + + protected async Task RequestStringWithCookies(string url, string cookieOverride = null, string referer = null) + { + var request = new Utils.Clients.WebRequest() + { + Url = url, + Type = RequestType.GET, + Cookies = cookieHeader, + Referer = referer + }; + + if (cookieOverride != null) + request.Cookies = cookieOverride; + return await webclient.GetString(request); + } + + protected async Task RequestBytesWithCookies(string url, string cookieOverride = null) + { + var request = new Utils.Clients.WebRequest() + { + Url = url, + Type = RequestType.GET, + Cookies = cookieOverride ?? cookieHeader + }; + + if (cookieOverride != null) + request.Cookies = cookieOverride; + return await webclient.GetBytes(request); + } + + protected async Task PostDataWithCookies(string url, Dictionary data, string cookieOverride = null) + { + var request = new Utils.Clients.WebRequest() + { + Url = url, + Type = RequestType.POST, + Cookies = cookieOverride ?? cookieHeader, + PostData = data + }; + return await webclient.GetString(request); + } + + protected async Task RequestLoginAndFollowRedirect(string url, Dictionary data, string cookies, bool returnCookiesFromFirstCall, string redirectUrlOverride = null, string referer =null) + { + var request = new Utils.Clients.WebRequest() + { + Url = url, + Type = RequestType.POST, + Cookies = cookies, + Referer = referer, + PostData = data + }; + var response = await webclient.GetString(request); + var firstCallCookies = response.Cookies; + await FollowIfRedirect(request, response, SiteLink, response.Cookies); + + if (returnCookiesFromFirstCall) + { + response.Cookies = firstCallCookies; + } + + return response; + } + + protected void ConfigureIfOK(string cookies, bool isLoggedin, Action onError) + { + if (isLoggedin) + { + cookieHeader = cookies; + SaveCookieHeaderAndConfigure(); + } else + { + onError(); + } + } } } diff --git a/src/Jackett/Indexers/BeyondHD.cs b/src/Jackett/Indexers/BeyondHD.cs index 152a9b9f9..889a85b58 100644 --- a/src/Jackett/Indexers/BeyondHD.cs +++ b/src/Jackett/Indexers/BeyondHD.cs @@ -2,6 +2,7 @@ using Jackett.Models; using Jackett.Services; using Jackett.Utils; +using Jackett.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using System; @@ -17,69 +18,42 @@ namespace Jackett.Indexers { public class BeyondHD : BaseIndexer, IIndexer { - private readonly string SearchUrl = ""; - private readonly string DownloadUrl = ""; + private string SearchUrl { get { return SiteLink + "browse.php?c40=1&c44=1&c48=1&c89=1&c46=1&c45=1&searchin=title&incldead=0&search={0}"; } } + private string DownloadUrl { get { return SiteLink + "download.php?torrent={0}"; } } - CookieContainer cookies; - HttpClientHandler handler; - HttpClient client; - - public BeyondHD(IIndexerManagerService i, Logger l) + public BeyondHD(IIndexerManagerService i, Logger l, IWebClient w) : base(name: "BeyondHD", description: "Without BeyondHD, your HDTV is just a TV", - link: new Uri("https://beyondhd.me"), + link: "https://beyondhd.me/", caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), manager: i, + client: w, logger: l) { - SearchUrl = SiteLink + "browse.php?c40=1&c44=1&c48=1&c89=1&c46=1&c45=1&searchin=title&incldead=0&search={0}"; - DownloadUrl = SiteLink + "download.php?torrent={0}"; - - cookies = new CookieContainer(); - handler = new HttpClientHandler - { - CookieContainer = cookies, - AllowAutoRedirect = true, - UseCookies = true, - }; - client = new HttpClient(handler); } public Task GetConfigurationForSetup() { - var config = new ConfigurationDataCookie(); - return Task.FromResult(config); + return Task.FromResult(new ConfigurationDataCookie()); } public async Task ApplyConfiguration(JToken configJson) { var config = new ConfigurationDataCookie(); config.LoadValuesFromJson(configJson); + cookieHeader = config.CookieHeader; - var jsonCookie = new JObject(); - jsonCookie["cookie_header"] = config.CookieHeader; - cookies.FillFromJson(SiteLink, jsonCookie, logger); - - var responseContent = await client.GetStringAsync(SiteLink); - - if (!responseContent.Contains("logout.php")) + var response = await webclient.GetString(new Utils.Clients.WebRequest() { - CQ dom = responseContent; + Url = SiteLink, + Cookies = cookieHeader + }); + + ConfigureIfOK(response.Cookies, response.Content.Contains("logout.php"), () => + { + CQ dom = response.Content; throw new ExceptionWithConfigData("Invalid cookie header", (ConfigurationData)config); - } - else - { - var configSaveData = new JObject(); - cookies.DumpToJson(SiteLink, configSaveData); - SaveConfig(configSaveData); - IsConfigured = true; - } - } - - public void LoadFromSavedConfiguration(JToken jsonConfig) - { - cookies.FillFromJson(SiteLink, jsonConfig, logger); - IsConfigured = true; + }); } public async Task PerformQuery(TorznabQuery query) @@ -88,11 +62,10 @@ namespace Jackett.Indexers var searchString = query.SanitizedSearchTerm + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = string.Format(SearchUrl, HttpUtility.UrlEncode(searchString)); - var results = await client.GetStringAsync(episodeSearchUrl); - + var results = await RequestStringWithCookies(episodeSearchUrl); try { - CQ dom = results; + CQ dom = results.Content; var rows = dom["table.torrenttable > tbody > tr.browse_color"]; foreach (var row in rows) { @@ -128,15 +101,10 @@ namespace Jackett.Indexers } catch (Exception ex) { - OnParseError(results, ex); + OnParseError(results.Content, ex); } return releases.ToArray(); } - - public Task Download(Uri link) - { - return client.GetByteArrayAsync(link); - } } } diff --git a/src/Jackett/Indexers/BitHdtv.cs b/src/Jackett/Indexers/BitHdtv.cs index 66ea5b035..c64af764f 100644 --- a/src/Jackett/Indexers/BitHdtv.cs +++ b/src/Jackett/Indexers/BitHdtv.cs @@ -2,6 +2,7 @@ using Jackett.Models; using Jackett.Services; using Jackett.Utils; +using Jackett.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using System; @@ -18,79 +19,46 @@ namespace Jackett.Indexers { public class BitHdtv : BaseIndexer, IIndexer { - private readonly string LoginUrl = ""; - private readonly string SearchUrl = ""; - private readonly string DownloadUrl = ""; + private string LoginUrl { get { return SiteLink + "takelogin.php"; } } + private string SearchUrl { get { return SiteLink + "torrents.php?cat=0&search="; } } + private string DownloadUrl { get { return SiteLink + "download.php?/{0}/dl.torrent"; } } - CookieContainer cookies; - HttpClientHandler handler; - HttpClient client; - - public BitHdtv(IIndexerManagerService i, Logger l) + public BitHdtv(IIndexerManagerService i, Logger l, IWebClient w) : base(name: "BIT-HDTV", description: "Home of high definition invites", - link: new Uri("https://www.bit-hdtv.com"), + link: "https://www.bit-hdtv.com/", caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), manager: i, + client: w, logger: l) { - LoginUrl = SiteLink + "takelogin.php"; - SearchUrl = SiteLink + "torrents.php?cat=0&search="; - DownloadUrl = SiteLink + "download.php?/{0}/dl.torrent"; - - cookies = new CookieContainer(); - handler = new HttpClientHandler - { - CookieContainer = cookies, - AllowAutoRedirect = true, - UseCookies = true, - }; - client = new HttpClient(handler); } public Task GetConfigurationForSetup() { - var config = new ConfigurationDataBasicLogin(); - return Task.FromResult(config); + return Task.FromResult(new ConfigurationDataBasicLogin()); } public async Task ApplyConfiguration(JToken configJson) { - var config = new ConfigurationDataBasicLogin(); - config.LoadValuesFromJson(configJson); + var incomingConfig = new ConfigurationDataBasicLogin(); + incomingConfig.LoadValuesFromJson(configJson); var pairs = new Dictionary { - { "username", config.Username.Value }, - { "password", config.Password.Value } + { "username", incomingConfig.Username.Value }, + { "password", incomingConfig.Password.Value } }; - var content = new FormUrlEncodedContent(pairs); - - var response = await client.PostAsync(LoginUrl, content); - var responseContent = await response.Content.ReadAsStringAsync(); - - if (!responseContent.Contains("logout.php")) + var response = await RequestLoginAndFollowRedirect(LoginUrl, pairs, null, true, null, SiteLink); + ConfigureIfOK(response.Cookies, response.Content != null && response.Content.Contains("logout.php"), () => { - CQ dom = responseContent; + CQ dom = response.Content; var messageEl = dom["table.detail td.text"].Last(); messageEl.Children("a").Remove(); messageEl.Children("style").Remove(); var errorMessage = messageEl.Text().Trim(); - throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)config); - } - else - { - var configSaveData = new JObject(); - cookies.DumpToJson(SiteLink, configSaveData); - SaveConfig(configSaveData); - IsConfigured = true; - } - } - - public void LoadFromSavedConfiguration(JToken jsonConfig) - { - cookies.FillFromJson(SiteLink, jsonConfig, logger); - IsConfigured = true; + throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)incomingConfig); + }); } public async Task PerformQuery(TorznabQuery query) @@ -99,10 +67,10 @@ namespace Jackett.Indexers var searchString = query.SanitizedSearchTerm + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = SearchUrl + HttpUtility.UrlEncode(searchString); - var results = await client.GetStringAsync(episodeSearchUrl); + var results = await RequestStringWithCookies(episodeSearchUrl); try { - CQ dom = results; + CQ dom = results.Content; dom["#needseed"].Remove(); var rows = dom["table[width='750'] > tbody"].Children(); foreach (var row in rows.Skip(1)) @@ -136,15 +104,10 @@ namespace Jackett.Indexers } catch (Exception ex) { - OnParseError(results, ex); + OnParseError(results.Content, ex); } return releases.ToArray(); } - - public Task Download(Uri link) - { - return client.GetByteArrayAsync(link); - } } } diff --git a/src/Jackett/Indexers/BitMeTV.cs b/src/Jackett/Indexers/BitMeTV.cs index 4ee4e8aad..bb1b981ce 100644 --- a/src/Jackett/Indexers/BitMeTV.cs +++ b/src/Jackett/Indexers/BitMeTV.cs @@ -1,7 +1,9 @@ using CsQuery; using Jackett.Models; +using Jackett.Models.IndexerConfig; using Jackett.Services; using Jackett.Utils; +using Jackett.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using System; @@ -19,68 +21,32 @@ namespace Jackett.Indexers { public class BitMeTV : BaseIndexer, IIndexer { - class BmtvConfig : ConfigurationData - { - public StringItem Username { get; private set; } + private string LoginUrl { get { return SiteLink + "login.php"; } } + private string LoginPost { get { return SiteLink + "takelogin.php"; } } + private string CaptchaUrl { get { return SiteLink + "visual.php"; } } + private string SearchUrl { get { return SiteLink + "browse.php"; } } - public StringItem Password { get; private set; } - - public ImageItem CaptchaImage { get; private set; } - - public StringItem CaptchaText { get; private set; } - - public BmtvConfig() - { - Username = new StringItem { Name = "Username" }; - Password = new StringItem { Name = "Password" }; - CaptchaImage = new ImageItem { Name = "Captcha Image" }; - CaptchaText = new StringItem { Name = "Captcha Text" }; - } - - public override Item[] GetItems() - { - return new Item[] { Username, Password, CaptchaImage, CaptchaText }; - } - } - - private readonly string LoginUrl = ""; - private readonly string LoginPost = ""; - private readonly string CaptchaUrl = ""; - private readonly string SearchUrl = ""; - - CookieContainer cookies; - HttpClientHandler handler; - HttpClient client; - - public BitMeTV(IIndexerManagerService i, Logger l) + public BitMeTV(IIndexerManagerService i, Logger l, IWebClient c) : base(name: "BitMeTV", description: "TV Episode specialty tracker", - link: new Uri("http://www.bitmetv.org"), + link: "http://www.bitmetv.org/", caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), manager: i, + client: c, logger: l) { - LoginUrl = SiteLink + "login.php"; - LoginPost = SiteLink + "takelogin.php"; - CaptchaUrl = SiteLink + "visual.php"; - SearchUrl = SiteLink + "browse.php"; - - cookies = new CookieContainer(); - handler = new HttpClientHandler - { - CookieContainer = cookies, - AllowAutoRedirect = true, - UseCookies = true, - }; - client = new HttpClient(handler); } public async Task GetConfigurationForSetup() { - await client.GetAsync(LoginUrl); - var captchaImage = await client.GetByteArrayAsync(CaptchaUrl); + var response = await webclient.GetString(new Utils.Clients.WebRequest() + { + Url = LoginUrl + }); + cookieHeader = response.Cookies; + var captchaImage = await RequestBytesWithCookies(CaptchaUrl); var config = new BmtvConfig(); - config.CaptchaImage.Value = captchaImage; + config.CaptchaImage.Value = captchaImage.Content; return (ConfigurationData)config; } @@ -95,34 +61,17 @@ namespace Jackett.Indexers { "secimage", config.CaptchaText.Value } }; - var content = new FormUrlEncodedContent(pairs); - - var response = await client.PostAsync(LoginPost, content); - var responseContent = await response.Content.ReadAsStringAsync(); - - if (!responseContent.Contains("/logout.php")) + var response = await RequestLoginAndFollowRedirect(LoginPost, pairs, cookieHeader, true); + ConfigureIfOK(cookieHeader, response.Content.Contains("/logout.php"), async () => { - CQ dom = responseContent; + CQ dom = response.Content; var messageEl = dom["table tr > td.embedded > h2"].Last(); var errorMessage = messageEl.Text(); - var captchaImage = await client.GetByteArrayAsync(CaptchaUrl); - config.CaptchaImage.Value = captchaImage; + var captchaImage = await RequestBytesWithCookies(CaptchaUrl); + config.CaptchaImage.Value = captchaImage.Content; config.CaptchaText.Value = ""; throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)config); - } - else - { - var configSaveData = new JObject(); - cookies.DumpToJson(SiteLink, configSaveData); - SaveConfig(configSaveData); - IsConfigured = true; - } - } - - public void LoadFromSavedConfiguration(JToken jsonConfig) - { - cookies.FillFromJson(SiteLink, jsonConfig, logger); - IsConfigured = true; + }); } public async Task PerformQuery(TorznabQuery query) @@ -131,10 +80,10 @@ namespace Jackett.Indexers var searchString = query.SanitizedSearchTerm + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = string.Format("{0}?search={1}&cat=0", SearchUrl, HttpUtility.UrlEncode(searchString)); - var results = await client.GetStringAsync(episodeSearchUrl); + var results = await RequestStringWithCookies(episodeSearchUrl); try { - CQ dom = results; + CQ dom = results.Content; var table = dom["tbody > tr > .latest"].Parent().Parent(); @@ -177,17 +126,10 @@ namespace Jackett.Indexers } catch (Exception ex) { - OnParseError(results, ex); + OnParseError(results.Content, ex); } return releases.ToArray(); - } - - public Task Download(Uri link) - { - return client.GetByteArrayAsync(link); - } - } } diff --git a/src/Jackett/Indexers/FrenchTorrentDb.cs b/src/Jackett/Indexers/FrenchTorrentDb.cs index 982fabd30..698250440 100644 --- a/src/Jackett/Indexers/FrenchTorrentDb.cs +++ b/src/Jackett/Indexers/FrenchTorrentDb.cs @@ -1,7 +1,9 @@ using CsQuery; using Jackett.Models; +using Jackett.Models.IndexerConfig; using Jackett.Services; using Jackett.Utils; +using Jackett.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using System; @@ -16,92 +18,41 @@ namespace Jackett.Indexers { class FrenchTorrentDb : BaseIndexer, IIndexer { - public event Action OnSaveConfigurationRequested; + private string MainUrl { get { return SiteLink + "?section=INDEX"; } } + private string SearchUrl { get { return SiteLink + "?section=TORRENTS&exact=1&name={0}&submit=GO"; } } - public event Action OnResultParsingError; - - class ConfigurationDataBasicLoginFrenchTorrentDb : ConfigurationData - { - public StringItem Cookie { get; private set; } - - public ConfigurationDataBasicLoginFrenchTorrentDb() - { - Cookie = new StringItem { Name = "Cookie" }; - } - - public override Item[] GetItems() - { - return new Item[] { Cookie }; - } - } - - - private readonly string MainUrl = ""; - private readonly string SearchUrl = ""; - - string cookie = string.Empty; - - CookieContainer cookies; - HttpClientHandler handler; - HttpClient client; - - public FrenchTorrentDb(IIndexerManagerService i, Logger l) + public FrenchTorrentDb(IIndexerManagerService i, Logger l, IWebClient c) : base(name: "FrenchTorrentDb", description: "One the biggest French Torrent Tracker", - link: new Uri("http://www.frenchtorrentdb.com/"), + link: "http://www.frenchtorrentdb.com/", caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), manager: i, + client: c, logger: l) { - MainUrl = SiteLink + "?section=INDEX"; - SearchUrl = SiteLink + "?section=TORRENTS&exact=1&name={0}&submit=GO"; - - cookies = new CookieContainer(); - handler = new HttpClientHandler - { - CookieContainer = cookies, - AllowAutoRedirect = true, - UseCookies = true, - }; - client = new HttpClient(handler); - client.DefaultRequestHeaders.UserAgent.ParseAdd(BrowserUtil.ChromeUserAgent); } public Task GetConfigurationForSetup() { - var config = new ConfigurationDataUrl(SiteLink); - return Task.FromResult(config); + return Task.FromResult(new ConfigurationDataUrl(SiteLink)); } public async Task ApplyConfiguration(Newtonsoft.Json.Linq.JToken configJson) { var config = new ConfigurationDataBasicLoginFrenchTorrentDb(); config.LoadValuesFromJson(configJson); - cookies.SetCookies(SiteLink, "WebsiteID=" + config.Cookie.Value); - var mainPage = await client.GetAsync(MainUrl); - string responseContent = await mainPage.Content.ReadAsStringAsync(); + var cookies = "WebsiteID=" + config.Cookie.Value; + var response = await webclient.GetString(new Utils.Clients.WebRequest() + { + Url = MainUrl, + Type = RequestType.GET, + Cookies = cookies + }); - if (!responseContent.Contains("/?section=LOGOUT")) + ConfigureIfOK(cookies, response.Content.Contains("/?section=LOGOUT"), () => { throw new ExceptionWithConfigData("Failed to login", (ConfigurationData)config); - } - else - { - var configSaveData = new JObject(); - configSaveData["cookie"] = config.Cookie.Value; - - if (OnSaveConfigurationRequested != null) - OnSaveConfigurationRequested(this, configSaveData); - - IsConfigured = true; - } - } - - public void LoadFromSavedConfiguration(Newtonsoft.Json.Linq.JToken jsonConfig) - { - cookie = (string)jsonConfig["cookie"]; - cookies.SetCookies(SiteLink, "WebsiteID=" + cookie); - IsConfigured = true; + }); } public async Task PerformQuery(TorznabQuery query) @@ -110,17 +61,10 @@ namespace Jackett.Indexers var searchString = query.SanitizedSearchTerm + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = string.Format(SearchUrl, HttpUtility.UrlEncode(searchString)); - - var message = new HttpRequestMessage(); - message.Method = HttpMethod.Get; - message.RequestUri = new Uri(episodeSearchUrl); - - var response = await client.SendAsync(message); - var results = await response.Content.ReadAsStringAsync(); + var response = await RequestStringWithCookies(episodeSearchUrl); try { - - CQ dom = results; + CQ dom = response.Cookies; var rows = dom[".results_index ul"]; foreach (var row in rows) { @@ -147,16 +91,10 @@ namespace Jackett.Indexers } catch (Exception ex) { - OnResultParsingError(this, results, ex); - throw ex; + OnParseError(response.Content, ex); } return releases.ToArray(); } - - public Task Download(Uri link) - { - return client.GetByteArrayAsync(link); - } } } diff --git a/src/Jackett/Indexers/Freshon.cs b/src/Jackett/Indexers/Freshon.cs index cc59ab6a4..7e021ad5b 100644 --- a/src/Jackett/Indexers/Freshon.cs +++ b/src/Jackett/Indexers/Freshon.cs @@ -3,6 +3,7 @@ using Jackett.Indexers; using Jackett.Models; using Jackett.Services; using Jackett.Utils; +using Jackett.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using System; @@ -21,94 +22,47 @@ namespace Jackett.Indexers { public class Freshon : BaseIndexer, IIndexer { - private readonly string LoginUrl = ""; - private readonly string LoginPostUrl = ""; - private readonly string SearchUrl = ""; + private string LoginUrl { get { return SiteLink + "login.php"; } } + private string LoginPostUrl { get { return SiteLink + "login.php?action=makelogin"; } } + private string SearchUrl { get { return SiteLink + "browse.php"; } } - CookieContainer cookies; - HttpClientHandler handler; - HttpClient client; - - public Freshon(IIndexerManagerService i, Logger l) + public Freshon(IIndexerManagerService i, Logger l, IWebClient c) : base(name: "FreshOnTV", description: "Our goal is to provide the latest stuff in the TV show domain", - link: new Uri("https://freshon.tv"), + link: "https://freshon.tv/", caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), manager: i, + client: c, logger: l) { - - LoginUrl = SiteLink + "login.php"; - LoginPostUrl = SiteLink + "login.php?action=makelogin"; - SearchUrl = SiteLink + "browse.php"; - - cookies = new CookieContainer(); - handler = new HttpClientHandler - { - CookieContainer = cookies, - AllowAutoRedirect = true, - UseCookies = true, - }; - client = new HttpClient(handler); } public async Task GetConfigurationForSetup() { - var request = CreateHttpRequest(new Uri(LoginUrl)); - var response = await client.SendAsync(request); - await response.Content.ReadAsStreamAsync(); - var config = new ConfigurationDataBasicLogin(); - return config; + return await Task.FromResult< ConfigurationData>(new ConfigurationDataBasicLogin()); } public async Task ApplyConfiguration(JToken configJson) { - var config = new ConfigurationDataBasicLogin(); - config.LoadValuesFromJson(configJson); - + var incomingConfig = new ConfigurationDataBasicLogin(); + incomingConfig.LoadValuesFromJson(configJson); var pairs = new Dictionary { - { "username", config.Username.Value }, - { "password", config.Password.Value } + { "username", incomingConfig.Username.Value }, + { "password", incomingConfig.Password.Value } }; - var content = new FormUrlEncodedContent(pairs); - var message = CreateHttpRequest(new Uri(LoginPostUrl)); - message.Method = HttpMethod.Post; - message.Content = content; - message.Headers.Referrer = new Uri(LoginUrl); + // Get inital cookies + cookieHeader = string.Empty; + var loginPage = await RequestStringWithCookies(LoginUrl); + var response = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, null, LoginUrl); - var response = await client.SendAsync(message); - var responseContent = await response.Content.ReadAsStringAsync(); - - if (!responseContent.Contains("/logout.php")) + ConfigureIfOK(response.Cookies, response.Content != null && response.Content.Contains("/logout.php"), () => { - CQ dom = responseContent; + CQ dom = response.Content; var messageEl = dom[".error_text"]; var errorMessage = messageEl.Text().Trim(); - throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)config); - } - else - { - var configSaveData = new JObject(); - cookies.DumpToJson(SiteLink, configSaveData); - SaveConfig(configSaveData); - IsConfigured = true; - } - } - - public void LoadFromSavedConfiguration(JToken jsonConfig) - { - cookies.FillFromJson(SiteLink, jsonConfig, logger); - IsConfigured = true; - } - - HttpRequestMessage CreateHttpRequest(Uri uri) - { - var message = new HttpRequestMessage(); - message.Method = HttpMethod.Get; - message.RequestUri = uri; - message.Headers.UserAgent.ParseAdd(BrowserUtil.ChromeUserAgent); - return message; + throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)incomingConfig); + }); } public async Task PerformQuery(TorznabQuery query) @@ -125,12 +79,10 @@ namespace Jackett.Indexers episodeSearchUrl = string.Format("{0}?search={1}&cat=0", SearchUrl, HttpUtility.UrlEncode(searchString)); } - var request = CreateHttpRequest(new Uri(episodeSearchUrl)); - var response = await client.SendAsync(request); - var results = await response.Content.ReadAsStringAsync(); + var results = await RequestStringWithCookies(episodeSearchUrl); try { - CQ dom = results; + CQ dom = results.Content; var rows = dom["#highlight > tbody > tr"]; @@ -170,18 +122,10 @@ namespace Jackett.Indexers } catch (Exception ex) { - OnParseError(results, ex); + OnParseError(results.Content, ex); } return releases.ToArray(); } - - public async Task Download(Uri link) - { - var request = CreateHttpRequest(link); - var response = await client.SendAsync(request); - var bytes = await response.Content.ReadAsByteArrayAsync(); - return bytes; - } } } diff --git a/src/Jackett/Indexers/HDSpace.cs b/src/Jackett/Indexers/HDSpace.cs index 53d9ef57e..9517b5f18 100644 --- a/src/Jackett/Indexers/HDSpace.cs +++ b/src/Jackett/Indexers/HDSpace.cs @@ -18,30 +18,23 @@ namespace Jackett.Indexers { public class HDSpace : BaseIndexer, IIndexer { - - private readonly string LoginUrl = ""; - private readonly string SearchUrl = ""; - private string cookieHeader = ""; - - private IWebClient webclient; + private string LoginUrl { get { return SiteLink + "index.php?page=login"; } } + private string SearchUrl { get { return SiteLink + "index.php?page=torrents&active=0&options=0&category=21%3B22&search={0}"; } } public HDSpace(IIndexerManagerService i, IWebClient wc, Logger l) : base(name: "HD-Space", description: "Sharing The Universe", - link: new Uri("https://hd-space.org"), + link: "https://hd-space.org/", caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), manager: i, + client: wc, logger: l) { - LoginUrl = SiteLink + "index.php?page=login"; - SearchUrl = SiteLink + "index.php?page=torrents&active=0&options=0&category=21%3B22&search={0}"; - webclient = wc; } public Task GetConfigurationForSetup() { - var config = new ConfigurationDataBasicLogin(); - return Task.FromResult(config); + return Task.FromResult(new ConfigurationDataBasicLogin()); } public async Task ApplyConfiguration(JToken configJson) @@ -49,11 +42,7 @@ namespace Jackett.Indexers var config = new ConfigurationDataBasicLogin(); config.LoadValuesFromJson(configJson); - var loginPage = await webclient.GetString(new WebRequest() - { - Url = LoginUrl, - Type = RequestType.GET - }); + var loginPage = await RequestStringWithCookies(LoginUrl, null); var pairs = new Dictionary { { "uid", config.Username.Value }, @@ -61,14 +50,7 @@ namespace Jackett.Indexers }; // Send Post - var loginPost = await webclient.GetString(new WebRequest() - { - Url = LoginUrl, - PostData = pairs, - Referer = LoginUrl, - Type = RequestType.POST, - Cookies = loginPage.Cookies - }); + var loginPost = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, null, LoginUrl); if (loginPost.Status == System.Net.HttpStatusCode.OK) { @@ -80,42 +62,12 @@ namespace Jackett.Indexers } // Get result from redirect - var loginResult = await webclient.GetString(new WebRequest() - { - Url = SiteLink + loginPost.RedirectingTo, - Type = RequestType.GET, - Cookies = loginPost.Cookies - }); + var loginResult = await RequestStringWithCookies(SiteLink + loginPost.RedirectingTo, loginPost.Cookies); - if (!loginResult.Content.Contains("logout.php")) + ConfigureIfOK(loginPost.Cookies, loginResult.Content.Contains("logout.php"), () => { throw new ExceptionWithConfigData("Login failed", (ConfigurationData)config); - } - else - { - cookieHeader = loginPost.Cookies; - var configSaveData = new JObject(); - configSaveData["cookies"] = cookieHeader; - SaveConfig(configSaveData); - IsConfigured = true; - } - - } - - public void LoadFromSavedConfiguration(JToken jsonConfig) - { - cookieHeader = (string)jsonConfig["cookies"]; - IsConfigured = true; - } - - public async Task Download(Uri link) - { - var response = await webclient.GetBytes(new WebRequest() - { - Url = link.ToString(), - Cookies = cookieHeader }); - return response.Content; } public async Task PerformQuery(TorznabQuery query) @@ -124,13 +76,7 @@ namespace Jackett.Indexers var searchString = query.SanitizedSearchTerm + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = string.Format(SearchUrl, HttpUtility.UrlEncode(searchString)); - - var response = await webclient.GetString(new WebRequest() - { - Url = episodeSearchUrl, - Referer = SiteLink.ToString(), - Cookies = cookieHeader - }); + var response = await RequestStringWithCookies(episodeSearchUrl); var results = response.Content; try diff --git a/src/Jackett/Indexers/HDTorrents.cs b/src/Jackett/Indexers/HDTorrents.cs index 9d8edc90d..f99c78379 100644 --- a/src/Jackett/Indexers/HDTorrents.cs +++ b/src/Jackett/Indexers/HDTorrents.cs @@ -2,6 +2,7 @@ using Jackett.Models; using Jackett.Services; using Jackett.Utils; +using Jackett.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using System; @@ -18,98 +19,50 @@ namespace Jackett.Indexers { public class HDTorrents : BaseIndexer, IIndexer { - private readonly string SearchUrl = ""; - private static string LoginUrl = ""; + private string SearchUrl { get { return SiteLink + "torrents.php?search={0}&active=1&options=0&category%5B%5D=59&category%5B%5D=60&category%5B%5D=30&category%5B%5D=38&page={1}"; } } + private string LoginUrl { get { return SiteLink + "login.php"; } } private const int MAXPAGES = 3; - CookieContainer cookies; - HttpClientHandler handler; - HttpClient client; - - public HDTorrents(IIndexerManagerService i, Logger l) + public HDTorrents(IIndexerManagerService i, Logger l, IWebClient w) : base(name: "HD-Torrents", description: "HD-Torrents is a private torrent website with HD torrents and strict rules on their content.", - link: new Uri("http://hdts.ru"),// Of the accessible domains the .ru seems the most reliable. https://hdts.ru | https://hd-torrents.org | https://hd-torrents.net | https://hd-torrents.me + link: "http://hdts.ru/",// Of the accessible domains the .ru seems the most reliable. https://hdts.ru | https://hd-torrents.org | https://hd-torrents.net | https://hd-torrents.me caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), manager: i, + client: w, logger: l) { - SearchUrl = SiteLink + "torrents.php?search={0}&active=1&options=0&category%5B%5D=59&category%5B%5D=60&category%5B%5D=30&category%5B%5D=38&page={1}"; - LoginUrl = SiteLink + "login.php"; - - cookies = new CookieContainer(); - handler = new HttpClientHandler - { - CookieContainer = cookies, - AllowAutoRedirect = true, - UseCookies = true, - }; - client = new HttpClient(handler); } public Task GetConfigurationForSetup() { - var config = new ConfigurationDataBasicLogin(); - return Task.FromResult(config); - } - - HttpRequestMessage CreateHttpRequest(string url) - { - var message = new HttpRequestMessage(); - message.Method = HttpMethod.Get; - message.RequestUri = new Uri(url); - message.Headers.UserAgent.ParseAdd(BrowserUtil.ChromeUserAgent); - return message; + return Task.FromResult(new ConfigurationDataBasicLogin()); } public async Task ApplyConfiguration(JToken configJson) { - var config = new ConfigurationDataBasicLogin(); - config.LoadValuesFromJson(configJson); - - var startMessage = CreateHttpRequest(LoginUrl); - var results = await (await client.SendAsync(startMessage)).Content.ReadAsStringAsync(); - + var incomingConfig = new ConfigurationDataBasicLogin(); + incomingConfig.LoadValuesFromJson(configJson); + var loginPage = await RequestStringWithCookies(LoginUrl, null); var pairs = new Dictionary { - { "uid", config.Username.Value }, - { "pwd", config.Password.Value } - }; + { "uid", incomingConfig.Username.Value }, + { "pwd", incomingConfig.Password.Value } + }; - var content = new FormUrlEncodedContent(pairs); + var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, null, LoginUrl); - var loginRequest = CreateHttpRequest(LoginUrl); - loginRequest.Method = HttpMethod.Post; - loginRequest.Content = content; - loginRequest.Headers.Referrer = new Uri("https://hd-torrents.org/torrents.php"); - - var response = await client.SendAsync(loginRequest); - var responseContent = await response.Content.ReadAsStringAsync(); - - if (!responseContent.Contains("If your browser doesn't have javascript enabled")) + ConfigureIfOK(result.Content, result.Content != null && result.Content.Contains("If your browser doesn't have javascript enabled"), () => { var errorMessage = "Couldn't login"; - throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)config); - } - else - { - var configSaveData = new JObject(); - cookies.DumpToJson(SiteLink, configSaveData); - SaveConfig(configSaveData); - IsConfigured = true; - } + throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)incomingConfig); + }); } - public void LoadFromSavedConfiguration(JToken jsonConfig) + public async Task PerformQuery(TorznabQuery query) { - cookies.FillFromJson(SiteLink, jsonConfig, logger); - IsConfigured = true; - } - - async Task PerformQuery(TorznabQuery query, Uri baseUrl) - { - List releases = new List(); - List searchurls = new List(); + var releases = new List(); + var searchurls = new List(); var searchString = query.SanitizedSearchTerm + " " + query.GetEpisodeSearchString(); for (int page = 0; page < MAXPAGES; page++) @@ -117,12 +70,12 @@ namespace Jackett.Indexers searchurls.Add(string.Format(SearchUrl, HttpUtility.UrlEncode(searchString.Trim()), page)); } - foreach (string SearchUrl in searchurls) + foreach (string searchUrl in searchurls) { - var results = await client.GetStringAsync(SearchUrl); + var results = await RequestStringWithCookies(searchUrl); try { - CQ dom = results; + CQ dom = results.Content; ReleaseInfo release; int rowCount = 0; @@ -182,21 +135,11 @@ namespace Jackett.Indexers } catch (Exception ex) { - OnParseError(results, ex); + OnParseError(results.Content, ex); } } return releases.ToArray(); } - - public async Task PerformQuery(TorznabQuery query) - { - return await PerformQuery(query, SiteLink); - } - - public Task Download(Uri link) - { - return client.GetByteArrayAsync(link); - } } } diff --git a/src/Jackett/Indexers/IIndexer.cs b/src/Jackett/Indexers/IIndexer.cs index cf6ba4c74..dc92bc3be 100644 --- a/src/Jackett/Indexers/IIndexer.cs +++ b/src/Jackett/Indexers/IIndexer.cs @@ -11,12 +11,12 @@ namespace Jackett.Indexers { public interface IIndexer { + string SiteLink { get; } + string DisplayName { get; } string DisplayDescription { get; } string ID { get; } - Uri SiteLink { get; } - TorznabCapabilities TorznabCaps { get; } // Whether this indexer has been configured, verified and saved in the past and has the settings required for functioning diff --git a/src/Jackett/Indexers/IPTorrents.cs b/src/Jackett/Indexers/IPTorrents.cs index f9b7a2de2..0bb690f91 100644 --- a/src/Jackett/Indexers/IPTorrents.cs +++ b/src/Jackett/Indexers/IPTorrents.cs @@ -19,82 +19,54 @@ namespace Jackett.Indexers { public class IPTorrents : BaseIndexer, IIndexer { - private readonly string SearchUrl = ""; - private string cookieHeader = ""; - - private IWebClient webclient; + private string SearchUrl { get { return SiteLink + "t?26=&55=&78=&23=&24=&25=&66=&82=&65=&83=&79=&22=&5=&4=&60=&q="; } } public IPTorrents(IIndexerManagerService i, IWebClient wc, Logger l) : base(name: "IPTorrents", description: "Always a step ahead.", - link: new Uri("https://iptorrents.com/"), + link: "https://iptorrents.com/", caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), manager: i, + client: wc, logger: l) { TorznabCaps.Categories.Add(new TorznabCategory { ID = "5070", Name = "TV/Anime" }); - SearchUrl = SiteLink + "t?26=&55=&78=&23=&24=&25=&66=&82=&65=&83=&79=&22=&5=&4=&60=&q="; - webclient = wc; } public Task GetConfigurationForSetup() { - var config = new ConfigurationDataBasicLogin(); - return Task.FromResult((ConfigurationData)config); + return Task.FromResult(new ConfigurationDataBasicLogin()); } public async Task ApplyConfiguration(JToken configJson) { - - var config = new ConfigurationDataBasicLogin(); - config.LoadValuesFromJson(configJson); - + var incomingConfig = new ConfigurationDataBasicLogin(); + incomingConfig.LoadValuesFromJson(configJson); var pairs = new Dictionary { - { "username", config.Username.Value }, - { "password", config.Password.Value } - }; - - var response = await webclient.GetString(new Utils.Clients.WebRequest() + { "username", incomingConfig.Username.Value }, + { "password", incomingConfig.Password.Value } + }; + var request = new Utils.Clients.WebRequest() { - Url = SiteLink.ToString(), - PostData = pairs, - Referer = SiteLink.ToString(), - Type = RequestType.POST - }); + Url = SiteLink, + Type = RequestType.POST, + Referer = SiteLink, + PostData = pairs + }; + var response = await webclient.GetString(request); + var firstCallCookies = response.Cookies; + // Redirect to ? + await FollowIfRedirect(request, response,null, firstCallCookies); + // Redirect to /t + await FollowIfRedirect(request, response, null, firstCallCookies); - cookieHeader = response.Cookies; - if (response.Status == HttpStatusCode.Found) + ConfigureIfOK(firstCallCookies, response.Content.Contains("/my.php"), () => { - response = await webclient.GetString(new Utils.Clients.WebRequest() - { - Url = SearchUrl, - Referer = SiteLink.ToString(), - Cookies = response.Cookies - }); - } - - var responseContent = response.Content; - - if (!responseContent.Contains("/my.php")) - { - CQ dom = responseContent; + CQ dom = response.Content; var messageEl = dom["body > div"].First(); var errorMessage = messageEl.Text().Trim(); - throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)config); - } - else - { - var configSaveData = new JObject(); - configSaveData["cookie_header"] = cookieHeader; - SaveConfig(configSaveData); - IsConfigured = true; - } - } - - public void LoadFromSavedConfiguration(JToken jsonConfig) - { - cookieHeader = (string)jsonConfig["cookie_header"]; - IsConfigured = true; + throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)incomingConfig); + }); } public async Task PerformQuery(TorznabQuery query) @@ -110,13 +82,7 @@ namespace Jackett.Indexers { try { - response = await webclient.GetString(new Utils.Clients.WebRequest() - { - Url = episodeSearchUrl, - Referer = SiteLink.ToString(), - Cookies = cookieHeader - }); - + response = await RequestStringWithCookies(episodeSearchUrl, null, SearchUrl); break; } catch (Exception e) @@ -172,16 +138,5 @@ namespace Jackett.Indexers return releases.ToArray(); } - - public async Task Download(Uri link) - { - var response = await webclient.GetBytes(new Utils.Clients.WebRequest() - { - Url = link.ToString(), - Cookies = cookieHeader - }); - - return response.Content; - } } } diff --git a/src/Jackett/Indexers/MoreThanTV.cs b/src/Jackett/Indexers/MoreThanTV.cs index 1d8c3199f..23039992d 100644 --- a/src/Jackett/Indexers/MoreThanTV.cs +++ b/src/Jackett/Indexers/MoreThanTV.cs @@ -19,33 +19,25 @@ namespace Jackett.Indexers { public class MoreThanTV : BaseIndexer, IIndexer { - private readonly string LoginUrl = ""; - private readonly string SearchUrl = ""; - private readonly string DownloadUrl = ""; - private readonly string GuidUrl = ""; - - private IWebClient client; - private string cookieHeader = ""; + private string LoginUrl { get { return SiteLink + "login.php"; } } + private string SearchUrl { get { return SiteLink + "ajax.php?action=browse&searchstr="; } } + private string DownloadUrl { get { return SiteLink + "torrents.php?action=download&id="; } } + private string GuidUrl { get { return SiteLink + "torrents.php?torrentid="; } } public MoreThanTV(IIndexerManagerService i, IWebClient c, Logger l) : base(name: "MoreThanTV", description: "ROMANIAN Private Torrent Tracker for TV / MOVIES, and the internal tracker for the release group DRACULA.", - link: new Uri("https://www.morethan.tv"), + link: "https://www.morethan.tv/", caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), manager: i, + client: c, logger: l) { - LoginUrl = SiteLink + "login.php"; - SearchUrl = SiteLink + "ajax.php?action=browse&searchstr="; - DownloadUrl = SiteLink + "torrents.php?action=download&id="; - GuidUrl = SiteLink + "torrents.php?torrentid="; - client = c; } public Task GetConfigurationForSetup() { - var config = new ConfigurationDataBasicLogin(); - return Task.FromResult(config); + return Task.FromResult(new ConfigurationDataBasicLogin()); } public async Task ApplyConfiguration(JToken configJson) @@ -58,45 +50,15 @@ namespace Jackett.Indexers { "login", "Log in" }, { "keeplogged", "1" } }; - - var loginResponse = await client.GetString(new Utils.Clients.WebRequest() - { - PostData = pairs, - Url = LoginUrl, - Type = RequestType.POST - }); - - if (loginResponse.Status == HttpStatusCode.Found) - { - cookieHeader = loginResponse.Cookies; - loginResponse = await client.GetString(new Utils.Clients.WebRequest() - { - Url = SiteLink.ToString(), - Cookies = cookieHeader - }); - } - if (!loginResponse.Content.Contains("logout.php?")) + var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, null, true, SearchUrl, SiteLink); + ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("logout.php?"), () => { - CQ dom = loginResponse.Content; + CQ dom = result.Content; dom["#loginform > table"].Remove(); var errorMessage = dom["#loginform"].Text().Trim().Replace("\n\t", " "); throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)config); - - } - else - { - var configSaveData = new JObject(); - configSaveData["cookie_header"] = cookieHeader; - SaveConfig(configSaveData); - IsConfigured = true; - } - } - - public void LoadFromSavedConfiguration(JToken jsonConfig) - { - cookieHeader = (string)jsonConfig["cookie_header"]; - IsConfigured = true; + }); } private void FillReleaseInfoFromJson(ReleaseInfo release, JObject r) @@ -123,13 +85,7 @@ namespace Jackett.Indexers { try { - response = await client.GetString(new Utils.Clients.WebRequest() - { - Url = episodeSearchUrl, - Type = RequestType.GET, - Cookies = cookieHeader - }); - + response = await RequestStringWithCookies(episodeSearchUrl); break; } catch (Exception e){ @@ -173,7 +129,6 @@ namespace Jackett.Indexers FillReleaseInfoFromJson(release, r); releases.Add(release); } - } } catch (Exception ex) @@ -183,17 +138,5 @@ namespace Jackett.Indexers return releases.ToArray(); } - - public async Task Download(Uri link) - { - var result = await client.GetBytes(new Utils.Clients.WebRequest() - { - Cookies = cookieHeader, - Url = link.ToString(), - Type = RequestType.GET - }); - - return result.Content; - } } } diff --git a/src/Jackett/Indexers/Pretome.cs b/src/Jackett/Indexers/Pretome.cs index 21329864c..14fbc8552 100644 --- a/src/Jackett/Indexers/Pretome.cs +++ b/src/Jackett/Indexers/Pretome.cs @@ -11,52 +11,30 @@ using NLog; using Jackett.Utils; using CsQuery; using System.Web; +using Jackett.Models.IndexerConfig; namespace Jackett.Indexers { public class Pretome : BaseIndexer, IIndexer { - - class PretomeConfiguration : ConfigurationDataBasicLogin - { - public StringItem Pin { get; private set; } - - public PretomeConfiguration() : base() - { - Pin = new StringItem { Name = "Login Pin Number" }; - } - - public override Item[] GetItems() - { - return new Item[] { Pin, Username, Password }; - } - } - - private readonly string LoginUrl = ""; - private readonly string LoginReferer = ""; - private readonly string SearchUrl = ""; - private string cookieHeader = ""; - - private IWebClient webclient; + private string LoginUrl { get { return SiteLink + "takelogin.php"; } } + private string LoginReferer { get { return SiteLink + "index.php?cat=1"; } } + private string SearchUrl { get { return SiteLink + "browse.php?tags=&st=1&tf=all&cat%5B%5D=7&search={0}"; } } public Pretome(IIndexerManagerService i, IWebClient wc, Logger l) : base(name: "PreToMe", description: "BitTorrent site for High Quality, High Definition (HD) movies and TV Shows", - link: new Uri("https://pretome.info/"), + link: "https://pretome.info/", caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), + client: wc, manager: i, logger: l) { - LoginUrl = SiteLink + "takelogin.php"; - LoginReferer = SiteLink + "index.php?cat=1"; - SearchUrl = SiteLink + "browse.php?tags=&st=1&tf=all&cat%5B%5D=7&search={0}"; - webclient = wc; } public Task GetConfigurationForSetup() { - var config = new PretomeConfiguration(); - return Task.FromResult(config); + return Task.FromResult(new PretomeConfiguration()); } public async Task ApplyConfiguration(JToken configJson) @@ -78,58 +56,20 @@ namespace Jackett.Indexers { "login", "Login" } }; - // Send Post - var loginPost = await webclient.GetString(new WebRequest() - { - Url = LoginUrl, - PostData = pairs, - Referer = LoginReferer, - Type = RequestType.POST, - Cookies = loginPage.Cookies - }); - + var loginPost = await PostDataWithCookies(LoginUrl, pairs, loginPage.Cookies); if (loginPost.RedirectingTo == null) { throw new ExceptionWithConfigData("Login failed. Did you use the PIN number that pretome emailed you?", (ConfigurationData)config); } // Get result from redirect - var loginResult = await webclient.GetString(new WebRequest() - { - Url = loginPost.RedirectingTo, - Type = RequestType.GET, - Cookies = loginPost.Cookies - }); + var loginResult = await RequestStringWithCookies(loginPost.RedirectingTo); - if (!loginResult.Content.Contains("logout.php")) + ConfigureIfOK(loginPost.Cookies, loginResult.Content != null && loginResult.Content.Contains("logout.php"), () => { throw new ExceptionWithConfigData("Failed", (ConfigurationData)config); - } - else - { - cookieHeader = loginPost.Cookies; - var configSaveData = new JObject(); - configSaveData["cookies"] = cookieHeader; - SaveConfig(configSaveData); - IsConfigured = true; - } - } - - public void LoadFromSavedConfiguration(JToken jsonConfig) - { - cookieHeader = (string)jsonConfig["cookies"]; - IsConfigured = true; - } - - public async Task Download(Uri link) - { - var response = await webclient.GetBytes(new WebRequest() - { - Url = link.ToString(), - Cookies = cookieHeader }); - return response.Content; } public async Task PerformQuery(TorznabQuery query) @@ -139,17 +79,11 @@ namespace Jackett.Indexers var searchString = query.SanitizedSearchTerm + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = string.Format(SearchUrl, HttpUtility.UrlEncode(searchString)); - var response = await webclient.GetString(new WebRequest() - { - Url = episodeSearchUrl, - Referer = SiteLink.ToString(), - Cookies = cookieHeader - }); - var results = response.Content; + var response = await RequestStringWithCookies(episodeSearchUrl); try { - CQ dom = results; + CQ dom = response.Content; var rows = dom["table > tbody > tr.browse"]; foreach (var row in rows) { @@ -186,7 +120,7 @@ namespace Jackett.Indexers } catch (Exception ex) { - OnParseError(results, ex); + OnParseError(response.Content, ex); } return releases.ToArray(); } diff --git a/src/Jackett/Indexers/PrivateHD.cs b/src/Jackett/Indexers/PrivateHD.cs index 6337b8b50..be22064ba 100644 --- a/src/Jackett/Indexers/PrivateHD.cs +++ b/src/Jackett/Indexers/PrivateHD.cs @@ -19,101 +19,46 @@ namespace Jackett.Indexers { public class PrivateHD : BaseIndexer, IIndexer { - private readonly string LoginUrl = ""; - private readonly string SearchUrl = ""; - private string cookieHeader = ""; - - private IWebClient webclient; + private string LoginUrl { get { return SiteLink + "auth/login"; } } + private string SearchUrl { get { return SiteLink + "torrents?in=1&type=2&search={0}"; } } public PrivateHD(IIndexerManagerService i, IWebClient wc, Logger l) : base(name: "PrivateHD", description: "BitTorrent site for High Quality, High Definition (HD) movies and TV Shows", - link: new Uri("https://privatehd.to"), + link: "https://privatehd.to/", caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), manager: i, + client: wc, logger: l) { - LoginUrl = SiteLink + "auth/login"; - SearchUrl = SiteLink + "torrents?in=1&type=2&search={0}"; - webclient = wc; } public async Task ApplyConfiguration(JToken configJson) { - var config = new ConfigurationDataBasicLogin(); - config.LoadValuesFromJson(configJson); - - var loginPage = await webclient.GetString(new Utils.Clients.WebRequest() - { - Url = LoginUrl, - Type = RequestType.GET - }); - + var incomingConfig = new ConfigurationDataBasicLogin(); + incomingConfig.LoadValuesFromJson(configJson); + var loginPage = await RequestStringWithCookies(LoginUrl); var token = new Regex("Avz.CSRF_TOKEN = '(.*?)';").Match(loginPage.Content).Groups[1].ToString(); var pairs = new Dictionary { { "_token", token }, - { "username_email", config.Username.Value }, - { "password", config.Password.Value }, + { "username_email", incomingConfig.Username.Value }, + { "password", incomingConfig.Password.Value }, { "remember", "on" } }; - // Send Post - var loginPost = await webclient.GetString(new Utils.Clients.WebRequest() + var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, null, true, null, LoginUrl); + ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("auth/logout"), () => { - Url = LoginUrl, - PostData = pairs, - Referer = LoginUrl, - Type = RequestType.POST, - Cookies = loginPage.Cookies - }); - - // Get result from redirect - var loginResult = await webclient.GetString(new Utils.Clients.WebRequest() - { - Url = loginPost.RedirectingTo, - Type = RequestType.GET, - Cookies = loginPost.Cookies - }); - - if (!loginResult.Content.Contains("auth/logout")) - { - CQ dom = loginResult.Content; + CQ dom = result.Content; var messageEl = dom[".form-error"]; var errorMessage = messageEl.Text().Trim(); - throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)config); - } - else - { - cookieHeader = loginPost.Cookies; - var configSaveData = new JObject(); - configSaveData["cookies"] = cookieHeader; - SaveConfig(configSaveData); - IsConfigured = true; - } - - } - - public async Task Download(Uri link) - { - var response = await webclient.GetBytes(new Utils.Clients.WebRequest() - { - Url = link.ToString(), - Cookies = cookieHeader + throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)incomingConfig); }); - - return response.Content; } public Task GetConfigurationForSetup() { - var config = new ConfigurationDataBasicLogin(); - return Task.FromResult(config); - } - - public void LoadFromSavedConfiguration(JToken jsonConfig) - { - cookieHeader = (string)jsonConfig["cookies"]; - IsConfigured = true; + return Task.FromResult(new ConfigurationDataBasicLogin()); } public async Task PerformQuery(TorznabQuery query) @@ -123,17 +68,11 @@ namespace Jackett.Indexers var searchString = query.SanitizedSearchTerm + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = string.Format(SearchUrl, HttpUtility.UrlEncode(searchString)); - var response = await webclient.GetString(new Utils.Clients.WebRequest() - { - Url = episodeSearchUrl, - Referer = SiteLink.ToString(), - Cookies = cookieHeader - }); - var results = response.Content; + var response = await RequestStringWithCookies(episodeSearchUrl); try { - CQ dom = results; + CQ dom = response.Content; var rows = dom["table > tbody > tr"]; foreach (var row in rows) { @@ -165,7 +104,7 @@ namespace Jackett.Indexers } catch (Exception ex) { - OnParseError(results, ex); + OnParseError(response.Content, ex); } return releases.ToArray(); } diff --git a/src/Jackett/Indexers/Rarbg.cs b/src/Jackett/Indexers/Rarbg.cs index 4d43927b1..f60f31d1c 100644 --- a/src/Jackett/Indexers/Rarbg.cs +++ b/src/Jackett/Indexers/Rarbg.cs @@ -2,6 +2,7 @@ using Jackett.Models; using Jackett.Services; using Jackett.Utils; +using Jackett.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using System; @@ -16,39 +17,26 @@ namespace Jackett.Indexers { public class Rarbg : BaseIndexer, IIndexer { - const string DefaultUrl = "http://torrentapi.org"; - const string TokenUrl = "/pubapi.php?get_token=get_token&format=json"; - const string SearchTVRageUrl = "/pubapi.php?mode=search&search_tvrage={0}&token={1}&format=json&min_seeders=1"; - const string SearchQueryUrl = "/pubapi.php?mode=search&search_string={0}&token={1}&format=json&min_seeders=1"; + private const string DefaultUrl = "http://torrentapi.org/"; + private const string TokenUrl = "pubapi.php?get_token=get_token&format=json"; + private const string SearchTVRageUrl = "pubapi.php?mode=search&search_tvrage={0}&token={1}&format=json&min_seeders=1"; + private const string SearchQueryUrl = "pubapi.php?mode=search&search_string={0}&token={1}&format=json&min_seeders=1"; private string BaseUrl; - CookieContainer cookies; - HttpClientHandler handler; - HttpClient client; - - - public Rarbg(IIndexerManagerService i, Logger l) + public Rarbg(IIndexerManagerService i, Logger l, IWebClient wc) : base(name: "RARBG", description: "RARBG", - link: new Uri("https://rarbg.com"), + link: "https://rarbg.com/", caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), manager: i, + client: wc, logger: l) { - cookies = new CookieContainer(); - handler = new HttpClientHandler - { - CookieContainer = cookies, - AllowAutoRedirect = true, - UseCookies = true, - }; - client = new HttpClient(handler); } public Task GetConfigurationForSetup() { - var config = new ConfigurationDataUrl(DefaultUrl); - return Task.FromResult(config); + return Task.FromResult(new ConfigurationDataUrl(DefaultUrl)); } public async Task ApplyConfiguration(JToken configJson) @@ -70,27 +58,16 @@ namespace Jackett.Indexers IsConfigured = true; } - public void LoadFromSavedConfiguration(JToken jsonConfig) + public override void LoadFromSavedConfiguration(JToken jsonConfig) { BaseUrl = (string)jsonConfig["base_url"]; - IsConfigured = true; - } - - HttpRequestMessage CreateHttpRequest(string uri) - { - var message = new HttpRequestMessage(); - message.Method = HttpMethod.Get; - message.RequestUri = new Uri(uri); - message.Headers.UserAgent.ParseAdd(BrowserUtil.ChromeUserAgent); - return message; + IsConfigured = !string.IsNullOrEmpty(BaseUrl); } async Task GetToken(string url) { - var request = CreateHttpRequest(url + TokenUrl); - var response = await client.SendAsync(request); - var result = await response.Content.ReadAsStringAsync(); - JObject obj = JObject.Parse(result); + var response = await RequestStringWithCookies(url + TokenUrl); + JObject obj = JObject.Parse(response.Content); return (string)obj["token"]; } @@ -101,9 +78,7 @@ namespace Jackett.Indexers async Task PerformQuery(TorznabQuery query, string baseUrl) { - - List releases = new List(); - + var releases = new List(); string token = await GetToken(baseUrl); string searchUrl; if (query.RageID != 0) @@ -111,12 +86,10 @@ namespace Jackett.Indexers else searchUrl = string.Format(baseUrl + SearchQueryUrl, query.SanitizedSearchTerm, token); - var request = CreateHttpRequest(searchUrl); - var response = await client.SendAsync(request); - var results = await response.Content.ReadAsStringAsync(); + var results = await RequestStringWithCookies(searchUrl); try { - var jItems = JArray.Parse(results); + var jItems = JArray.Parse(results.Content); foreach (JObject item in jItems) { var release = new ReleaseInfo(); @@ -134,12 +107,12 @@ namespace Jackett.Indexers } catch (Exception ex) { - OnParseError(results, ex); + OnParseError(results.Content, ex); } return releases.ToArray(); } - public Task Download(Uri link) + public override Task Download(Uri link) { throw new NotImplementedException(); } diff --git a/src/Jackett/Indexers/SceneAccess.cs b/src/Jackett/Indexers/SceneAccess.cs index 176aed44e..a3752c409 100644 --- a/src/Jackett/Indexers/SceneAccess.cs +++ b/src/Jackett/Indexers/SceneAccess.cs @@ -17,29 +17,23 @@ namespace Jackett.Indexers { class SceneAccess : BaseIndexer, IIndexer { - private readonly string LoginUrl = ""; - private readonly string SearchUrl = ""; - - private IWebClient webclient; - private string cookieHeader = ""; + private string LoginUrl { get { return SiteLink + "login"; } } + private string SearchUrl { get { return SiteLink + "{0}?method=1&c{1}=1&search={2}"; } } public SceneAccess(IIndexerManagerService i, IWebClient c, Logger l) : base(name: "SceneAccess", description: "Your gateway to the scene", - link: new Uri("https://sceneaccess.eu/"), + link: "https://sceneaccess.eu/", caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), manager: i, + client: c, logger: l) { - LoginUrl = SiteLink + "login"; - SearchUrl = SiteLink + "{0}?method=1&c{1}=1&search={2}"; - webclient = c; } public Task GetConfigurationForSetup() { - var config = new ConfigurationDataBasicLogin(); - return Task.FromResult(config); + return Task.FromResult(new ConfigurationDataBasicLogin()); } public async Task ApplyConfiguration(JToken configJson) @@ -53,72 +47,24 @@ namespace Jackett.Indexers { "submit", "come on in" } }; - var content = new FormUrlEncodedContent(pairs); - - // Do the login - var response = await webclient.GetString(new Utils.Clients.WebRequest() + var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, null, true, SiteLink, LoginUrl); + ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("nav_profile"), () => { - PostData = pairs, - Referer = LoginUrl, - Type = RequestType.POST, - Url = LoginUrl - }); - - cookieHeader = response.Cookies; - - if (response.Status == HttpStatusCode.Found) - { - response = await webclient.GetString(new Utils.Clients.WebRequest() - { - Url = SiteLink.ToString(), - Referer = LoginUrl.ToString(), - Cookies = cookieHeader - }); - } - - if (!response.Content.Contains("nav_profile")) - { - CQ dom = response.Content; + CQ dom = result.Content; var messageEl = dom["#login_box_desc"]; var errorMessage = messageEl.Text().Trim(); throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)config); - } - else - { - var configSaveData = new JObject(); - configSaveData["cookie_header"] = cookieHeader; - SaveConfig(configSaveData); - IsConfigured = true; - } - } - - public void LoadFromSavedConfiguration(JToken jsonConfig) - { - cookieHeader = (string)jsonConfig["cookie_header"]; - if (!string.IsNullOrEmpty(cookieHeader)) - { - IsConfigured = true; - } + }); } public async Task PerformQuery(TorznabQuery query) { - List releases = new List(); - + var releases = new List(); var searchString = query.SanitizedSearchTerm + " " + query.GetEpisodeSearchString(); var searchSection = string.IsNullOrEmpty(query.Episode) ? "archive" : "browse"; var searchCategory = string.IsNullOrEmpty(query.Episode) ? "26" : "27"; - var searchUrl = string.Format(SearchUrl, searchSection, searchCategory, searchString); - - - var results = await webclient.GetString(new Utils.Clients.WebRequest() - { - Referer = LoginUrl, - Cookies = cookieHeader, - Type = RequestType.GET, - Url = searchUrl - }); + var results = await RequestStringWithCookies(searchUrl); try { @@ -160,16 +106,5 @@ namespace Jackett.Indexers return releases.ToArray(); } - - public async Task Download(Uri link) - { - var response = await webclient.GetBytes(new Utils.Clients.WebRequest() - { - Url = link.ToString(), - Cookies = cookieHeader - }); - - return response.Content; - } - } } +} diff --git a/src/Jackett/Indexers/SceneTime.cs b/src/Jackett/Indexers/SceneTime.cs index a7cb9aa1a..fc3405b99 100644 --- a/src/Jackett/Indexers/SceneTime.cs +++ b/src/Jackett/Indexers/SceneTime.cs @@ -2,6 +2,7 @@ using Jackett.Models; using Jackett.Services; using Jackett.Utils; +using Jackett.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using System; @@ -18,102 +19,62 @@ namespace Jackett.Indexers { public class SceneTime : BaseIndexer, IIndexer { - private readonly string LoginUrl = ""; - private readonly string SearchUrl = ""; - private readonly string DownloadUrl = ""; + private string LoginUrl { get { return SiteLink + "takelogin.php"; } } + private string SearchUrl { get { return SiteLink + "browse_API.php"; } } + private string DownloadUrl { get { return SiteLink + "download.php/{0}/download.torrent"; } } - CookieContainer cookies; - HttpClientHandler handler; - HttpClient client; - - public SceneTime(IIndexerManagerService i, Logger l) + public SceneTime(IIndexerManagerService i, Logger l, IWebClient w) : base(name: "SceneTime", description: "Always on time", - link: new Uri("https://www.scenetime.com/"), + link: "https://www.scenetime.com/", caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), manager: i, + client: w, logger: l) { - LoginUrl = SiteLink + "takelogin.php"; - SearchUrl = SiteLink + "browse_API.php"; - DownloadUrl = SiteLink + "download.php/{0}/download.torrent"; - - cookies = new CookieContainer(); - handler = new HttpClientHandler - { - CookieContainer = cookies, - AllowAutoRedirect = true, - UseCookies = true, - }; - client = new HttpClient(handler); } public Task GetConfigurationForSetup() { - var config = new ConfigurationDataBasicLogin(); - return Task.FromResult(config); + return Task.FromResult(new ConfigurationDataBasicLogin()); } public async Task ApplyConfiguration(JToken configJson) { - var config = new ConfigurationDataBasicLogin(); - config.LoadValuesFromJson(configJson); - + var incomingConfig = new ConfigurationDataBasicLogin(); + incomingConfig.LoadValuesFromJson(configJson); var pairs = new Dictionary { - { "username", config.Username.Value }, - { "password", config.Password.Value } + { "username", incomingConfig.Username.Value }, + { "password", incomingConfig.Password.Value } }; - var content = new FormUrlEncodedContent(pairs); - - var response = await client.PostAsync(LoginUrl, content); - var responseContent = await response.Content.ReadAsStringAsync(); - - if (!responseContent.Contains("logout.php")) + var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, null, true, null, LoginUrl); + ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("logout.php"), () => { - CQ dom = responseContent; + CQ dom = result.Content; var errorMessage = dom["td.text"].Text().Trim(); - throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)config); - } - else - { - var configSaveData = new JObject(); - cookies.DumpToJson(SiteLink, configSaveData); - SaveConfig(configSaveData); - IsConfigured = true; - } + throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)incomingConfig); + }); } - public void LoadFromSavedConfiguration(JToken jsonConfig) + private Dictionary GetSearchFormData(string searchString) { - cookies.FillFromJson(SiteLink, jsonConfig, logger); - IsConfigured = true; - } - - FormUrlEncodedContent GetSearchFormData(string searchString) - { - var pairs = new Dictionary { + return new Dictionary { { "c2", "1" }, { "c43", "1" }, { "c9", "1" }, { "c63", "1" }, { "c77", "1" }, { "c100", "1" }, { "c101", "1" }, { "cata", "yes" }, { "sec", "jax" }, { "search", searchString} }; - var content = new FormUrlEncodedContent(pairs); - return content; } public async Task PerformQuery(TorznabQuery query) { - List releases = new List(); - + var releases = new List(); var searchString = query.SanitizedSearchTerm + " " + query.GetEpisodeSearchString(); - - var searchContent = GetSearchFormData(searchString); - var response = await client.PostAsync(SearchUrl, searchContent); - var results = await response.Content.ReadAsStringAsync(); + var results = await PostDataWithCookies(SearchUrl, GetSearchFormData(searchString)); try { - CQ dom = results; + CQ dom = results.Content; var rows = dom["tr.browse"]; foreach (var row in rows) { @@ -148,14 +109,9 @@ namespace Jackett.Indexers } catch (Exception ex) { - OnParseError(results, ex); + OnParseError(results.Content, ex); } return releases.ToArray(); } - - public Task Download(Uri link) - { - return client.GetByteArrayAsync(link); - } } } diff --git a/src/Jackett/Indexers/ShowRSS.cs b/src/Jackett/Indexers/ShowRSS.cs index 6bb4f644b..84aad0787 100644 --- a/src/Jackett/Indexers/ShowRSS.cs +++ b/src/Jackett/Indexers/ShowRSS.cs @@ -1,6 +1,7 @@ using Jackett.Models; using Jackett.Services; using Jackett.Utils; +using Jackett.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using System; @@ -18,37 +19,23 @@ namespace Jackett.Indexers { public class ShowRSS : BaseIndexer, IIndexer { - private readonly string searchAllUrl = ""; - string BaseUrl; + private string searchAllUrl { get { return SiteLink + "feeds/all.rss"; } } + private string BaseUrl; - CookieContainer cookies; - HttpClientHandler handler; - HttpClient client; - - public ShowRSS(IIndexerManagerService i, Logger l) + public ShowRSS(IIndexerManagerService i, Logger l, IWebClient wc) : base(name: "ShowRSS", description: "showRSS is a service that allows you to keep track of your favorite TV shows", - link: new Uri("http://showrss.info"), + link: "http://showrss.info/", caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), manager: i, + client: wc, logger: l) { - searchAllUrl = SiteLink + "feeds/all.rss"; - - cookies = new CookieContainer(); - handler = new HttpClientHandler - { - CookieContainer = cookies, - AllowAutoRedirect = true, - UseCookies = true, - }; - client = new HttpClient(handler); } public Task GetConfigurationForSetup() { - var config = new ConfigurationDataUrl(SiteLink); - return Task.FromResult(config); + return Task.FromResult(new ConfigurationDataUrl(SiteLink)); } public async Task ApplyConfiguration(Newtonsoft.Json.Linq.JToken configJson) @@ -69,7 +56,7 @@ namespace Jackett.Indexers IsConfigured = true; } - public void LoadFromSavedConfiguration(Newtonsoft.Json.Linq.JToken jsonConfig) + public override void LoadFromSavedConfiguration(Newtonsoft.Json.Linq.JToken jsonConfig) { BaseUrl = (string)jsonConfig["base_url"]; IsConfigured = true; @@ -80,39 +67,22 @@ namespace Jackett.Indexers return await PerformQuery(query, BaseUrl); } - public Task Download(Uri link) + public override Task Download(Uri link) { throw new NotImplementedException(); } - private WebClient getWebClient() - { - WebClient wc = new WebClient(); - WebHeaderCollection headers = new WebHeaderCollection(); - headers.Add("User-Agent", BrowserUtil.ChromeUserAgent); - wc.Headers = headers; - return wc; - } - async Task PerformQuery(TorznabQuery query, string baseUrl) { - List releases = new List(); - + var releases = new List(); var searchString = query.SanitizedSearchTerm + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = string.Format(searchAllUrl); - - XmlDocument xmlDoc = new XmlDocument(); - string xml = string.Empty; - WebClient wc = getWebClient(); + var result = await RequestStringWithCookies(episodeSearchUrl, string.Empty); + var xmlDoc = new XmlDocument(); try { - using (wc) - { - xml = await wc.DownloadStringTaskAsync(new Uri(episodeSearchUrl)); - xmlDoc.LoadXml(xml); - } - + xmlDoc.LoadXml(result.Content); ReleaseInfo release; string serie_title; @@ -143,7 +113,7 @@ namespace Jackett.Indexers } catch (Exception ex) { - OnParseError(xml, ex); + OnParseError(result.Content, ex); } return releases.ToArray(); diff --git a/src/Jackett/Indexers/SpeedCD.cs b/src/Jackett/Indexers/SpeedCD.cs index ba48e7161..47d8a76cc 100644 --- a/src/Jackett/Indexers/SpeedCD.cs +++ b/src/Jackett/Indexers/SpeedCD.cs @@ -2,6 +2,7 @@ using Jackett.Models; using Jackett.Services; using Jackett.Utils; +using Jackett.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using System; @@ -19,96 +20,56 @@ namespace Jackett.Indexers { public class SpeedCD : BaseIndexer, IIndexer { - private readonly string LoginUrl = ""; - private readonly string SearchUrl = ""; - private readonly string SearchFormData = "c53=1&c49=1&c2=1&c52=1&c41=1&c50=1&c30=1&jxt=4&jxw=b"; - private readonly string CommentsUrl = ""; - private readonly string DownloadUrl = ""; - - CookieContainer cookies; - HttpClientHandler handler; - HttpClient client; - - public SpeedCD(IIndexerManagerService i, Logger l) + private string LoginUrl { get { return SiteLink + "take_login.php"; } } + private string SearchUrl { get { return SiteLink + "V3/API/API.php"; } } + private string SearchFormData { get { return "c53=1&c49=1&c2=1&c52=1&c41=1&c50=1&c30=1&jxt=4&jxw=b"; } } + private string CommentsUrl { get { return SiteLink + "t/{0}"; } } + private string DownloadUrl { get { return SiteLink + "download.php?torrent={0}"; } } + + public SpeedCD(IIndexerManagerService i, Logger l, IWebClient wc) : base(name: "Speed.cd", description: "Your home now!", - link: new Uri("http://speed.cd"), + link: "http://speed.cd/", caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), manager: i, + client: wc, logger: l) { - LoginUrl = SiteLink + "take_login.php"; - SearchUrl = SiteLink + "V3/API/API.php"; - CommentsUrl = SiteLink + "t/{0}"; - DownloadUrl = SiteLink + "download.php?torrent={0}"; - - cookies = new CookieContainer(); - handler = new HttpClientHandler - { - CookieContainer = cookies, - AllowAutoRedirect = true, - UseCookies = true, - }; - client = new HttpClient(handler); } public Task GetConfigurationForSetup() { - var config = new ConfigurationDataBasicLogin(); - return Task.FromResult(config); + return Task.FromResult(new ConfigurationDataBasicLogin()); } public async Task ApplyConfiguration(JToken configJson) { - var config = new ConfigurationDataBasicLogin(); - config.LoadValuesFromJson(configJson); - + var incomingConfig = new ConfigurationDataBasicLogin(); + incomingConfig.LoadValuesFromJson(configJson); var pairs = new Dictionary { - { "username", config.Username.Value }, - { "password", config.Password.Value }, + { "username", incomingConfig.Username.Value }, + { "password", incomingConfig.Password.Value }, }; - var content = new FormUrlEncodedContent(pairs); - - var response = await client.PostAsync(LoginUrl, content); - var responseContent = await response.Content.ReadAsStringAsync(); - - if (!responseContent.Contains("logout.php")) + var result = await RequestLoginAndFollowRedirect(SiteLink, pairs, null, true, null, SiteLink); + ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("logout.php"), () => { - CQ dom = responseContent; + CQ dom = result.Content; var errorMessage = dom["h5"].First().Text().Trim(); - throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)config); - } - else - { - var configSaveData = new JObject(); - cookies.DumpToJson(SiteLink, configSaveData); - SaveConfig(configSaveData); - IsConfigured = true; - } - } - - public void LoadFromSavedConfiguration(JToken jsonConfig) - { - cookies.FillFromJson(SiteLink, jsonConfig, logger); - IsConfigured = true; + throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)incomingConfig); + }); } public async Task PerformQuery(TorznabQuery query) { - List releases = new List(); - + var releases = new List(); var formData = HttpUtility.ParseQueryString(SearchFormData); var formDict = formData.AllKeys.ToDictionary(t => t, t => formData[t]); formDict.Add("search", query.SanitizedSearchTerm); - var content = new FormUrlEncodedContent(formDict); - - var response = await client.PostAsync(SearchUrl, content); - var results = await response.Content.ReadAsStringAsync(); - + var response = await PostDataWithCookies(SearchUrl, formDict); try { - var jsonResult = JObject.Parse(results); + var jsonResult = JObject.Parse(response.Content); var resultArray = ((JArray)jsonResult["Fs"])[0]["Cn"]["torrents"]; foreach (var jobj in resultArray) { @@ -138,14 +99,9 @@ namespace Jackett.Indexers } catch (Exception ex) { - OnParseError(results, ex); + OnParseError(response.Content, ex); } return releases.ToArray(); } - - public Task Download(Uri link) - { - return client.GetByteArrayAsync(link); - } } } diff --git a/src/Jackett/Indexers/Strike.cs b/src/Jackett/Indexers/Strike.cs index 5c63476f2..1dea1605b 100644 --- a/src/Jackett/Indexers/Strike.cs +++ b/src/Jackett/Indexers/Strike.cs @@ -1,6 +1,7 @@ using Jackett.Models; using Jackett.Services; using Jackett.Utils; +using Jackett.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using System; @@ -17,74 +18,56 @@ namespace Jackett.Indexers { public class Strike : BaseIndexer, IIndexer { - private readonly string DownloadUrl = "/torrents/api/download/{0}.torrent"; - private readonly string SearchUrl = "/api/v2/torrents/search/?category=TV&phrase={0}"; - private string BaseUrl; + private string DownloadUrl { get { return baseUrl + "torrents/api/download/{0}.torrent"; } } + private string SearchUrl { get { return baseUrl + "api/v2/torrents/search/?category=TV&phrase={0}"; } } + private string baseUrl = null; - private CookieContainer cookies; - private HttpClientHandler handler; - private HttpClient client; - - public Strike(IIndexerManagerService i, Logger l) + public Strike(IIndexerManagerService i, Logger l, IWebClient wc) : base(name: "Strike", description: "Torrent search engine", - link: new Uri("https://getstrike.net"), + link: "https://getstrike.net/", caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), manager: i, + client: wc, logger: l) { - cookies = new CookieContainer(); - handler = new HttpClientHandler - { - CookieContainer = cookies, - AllowAutoRedirect = true, - UseCookies = true, - }; - client = new HttpClient(handler); } public Task GetConfigurationForSetup() { - var config = new ConfigurationDataUrl(SiteLink); - return Task.FromResult(config); + return Task.FromResult(new ConfigurationDataUrl(SiteLink)); } - public async Task ApplyConfiguration(JToken configJson) + public Task ApplyConfiguration(JToken configJson) { var config = new ConfigurationDataUrl(SiteLink); config.LoadValuesFromJson(configJson); - - var formattedUrl = config.GetFormattedHostUrl(); - var releases = await PerformQuery(new TorznabQuery(), formattedUrl); - if (releases.Length == 0) - throw new Exception("Could not find releases from this URL"); - - BaseUrl = formattedUrl; - + baseUrl = config.GetFormattedHostUrl(); var configSaveData = new JObject(); - configSaveData["base_url"] = BaseUrl; + configSaveData["base_url"] = baseUrl; SaveConfig(configSaveData); IsConfigured = true; + return Task.FromResult(0); } - public void LoadFromSavedConfiguration(JToken jsonConfig) + public override void LoadFromSavedConfiguration(JToken jsonConfig) { - BaseUrl = (string)jsonConfig["base_url"]; - IsConfigured = true; + baseUrl = (string)jsonConfig["base_url"]; + IsConfigured = !string.IsNullOrEmpty(baseUrl); } - public async Task PerformQuery(TorznabQuery query, string baseUrl) + public async Task PerformQuery(TorznabQuery query) { List releases = new List(); var searchTerm = string.IsNullOrEmpty(query.SanitizedSearchTerm) ? "2015" : query.SanitizedSearchTerm; var searchString = searchTerm + " " + query.GetEpisodeSearchString(); - var episodeSearchUrl = baseUrl + string.Format(SearchUrl, HttpUtility.UrlEncode(searchString.Trim())); - var results = await client.GetStringAsync(episodeSearchUrl); + var episodeSearchUrl =string.Format(SearchUrl, HttpUtility.UrlEncode(searchString.Trim())); + var results = await RequestStringWithCookies(episodeSearchUrl, string.Empty); try { - var jResults = JObject.Parse(results); + var jResults = JObject.Parse(results.Content); foreach (JObject result in (JArray)jResults["torrents"]) { var release = new ReleaseInfo(); @@ -112,25 +95,20 @@ namespace Jackett.Indexers release.InfoHash = (string)result["torrent_hash"]; release.MagnetUri = new Uri((string)result["magnet_uri"]); - release.Link = new Uri(string.Format("{0}{1}", baseUrl, string.Format(DownloadUrl, release.InfoHash))); + release.Link = new Uri(string.Format(DownloadUrl, release.InfoHash)); releases.Add(release); } } catch (Exception ex) { - OnParseError(results, ex); + OnParseError(results.Content, ex); } return releases.ToArray(); } - public async Task PerformQuery(TorznabQuery query) - { - return await PerformQuery(query, BaseUrl); - } - - public Task Download(Uri link) + public override Task Download(Uri link) { throw new NotImplementedException(); } diff --git a/src/Jackett/Indexers/T411.cs b/src/Jackett/Indexers/T411.cs index 814753558..797ecdcd4 100644 --- a/src/Jackett/Indexers/T411.cs +++ b/src/Jackett/Indexers/T411.cs @@ -1,6 +1,7 @@ using Jackett.Models; using Jackett.Services; using Jackett.Utils; +using Jackett.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using System; @@ -32,12 +33,13 @@ namespace Jackett.Indexers string token = string.Empty; DateTime lastTokenFetch = DateTime.MinValue; - public T411(IIndexerManagerService i, Logger l) + public T411(IIndexerManagerService i, Logger l,IWebClient wc) : base(name: "T411", description: "French Torrent Tracker", - link: new Uri("http://www.t411.io"), + link: "http://www.t411.io/", caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), manager: i, + client: wc, logger: l) { CommentsUrl = SiteLink + "/torrents/{0}"; @@ -107,7 +109,7 @@ namespace Jackett.Indexers IsConfigured = true; } - public void LoadFromSavedConfiguration(JToken jsonConfig) + public override void LoadFromSavedConfiguration(JToken jsonConfig) { username = (string)jsonConfig["username"]; password = (string)jsonConfig["password"]; @@ -167,7 +169,7 @@ namespace Jackett.Indexers return releases.ToArray(); } - public async Task Download(Uri link) + public override async Task Download(Uri link) { var message = new HttpRequestMessage(); message.Method = HttpMethod.Get; diff --git a/src/Jackett/Indexers/ThePirateBay.cs b/src/Jackett/Indexers/ThePirateBay.cs index e30169b19..598022251 100644 --- a/src/Jackett/Indexers/ThePirateBay.cs +++ b/src/Jackett/Indexers/ThePirateBay.cs @@ -2,6 +2,7 @@ using Jackett.Models; using Jackett.Services; using Jackett.Utils; +using Jackett.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using System; @@ -19,37 +20,25 @@ namespace Jackett.Indexers { public class ThePirateBay : BaseIndexer, IIndexer { - const string SearchUrl = "/search/{0}/0/99/208,205"; - string BaseUrl; + private const string SearchUrl = "/search/{0}/0/99/208,205"; + private string BaseUrl; - CookieContainer cookies; - HttpClientHandler handler; - HttpClient client; - - public ThePirateBay(IIndexerManagerService i, Logger l) + public ThePirateBay(IIndexerManagerService i, Logger l, IWebClient wc) : base(name: "The Pirate Bay", description: "The worlds largest bittorrent indexer", - link: new Uri("https://thepiratebay.mn"), + link: "https://thepiratebay.mn/", caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), manager: i, + client: wc, logger: l) { BaseUrl = SiteLink.ToString(); IsConfigured = false; - cookies = new CookieContainer(); - handler = new HttpClientHandler - { - CookieContainer = cookies, - AllowAutoRedirect = true, - UseCookies = true, - }; - client = new HttpClient(handler); } public Task GetConfigurationForSetup() { - var config = new ConfigurationDataUrl(BaseUrl); - return Task.FromResult(config); + return Task.FromResult(new ConfigurationDataUrl(BaseUrl)); } public async Task ApplyConfiguration(JToken configJson) @@ -70,7 +59,7 @@ namespace Jackett.Indexers IsConfigured = true; } - public void LoadFromSavedConfiguration(JToken jsonConfig) + public override void LoadFromSavedConfiguration(JToken jsonConfig) { BaseUrl = (string)jsonConfig["base_url"]; IsConfigured = true; @@ -83,27 +72,15 @@ namespace Jackett.Indexers async Task PerformQuery(TorznabQuery query, string baseUrl) { - List releases = new List(); - + var releases = new List(); var searchString = query.SanitizedSearchTerm + " " + query.GetEpisodeSearchString(); var queryStr = HttpUtility.UrlEncode(searchString); var episodeSearchUrl = baseUrl + string.Format(SearchUrl, queryStr); - - string results; - - if (Engine.IsWindows) - { - results = await client.GetStringAsync(episodeSearchUrl); - } - else - { - var response = await CurlHelper.GetAsync(episodeSearchUrl, null, episodeSearchUrl); - results = Encoding.UTF8.GetString(response.Content); - } + var response = await RequestStringWithCookies(episodeSearchUrl, string.Empty); try { - CQ dom = results; + CQ dom = response.Content; var rows = dom["#searchResult > tbody > tr"]; foreach (var row in rows) @@ -162,18 +139,14 @@ namespace Jackett.Indexers } catch (Exception ex) { - // OnResultParsingError(this, results, ex); - throw ex; + OnParseError(response.Content, ex); } return releases.ToArray(); } - - public Task Download(Uri link) + public override Task Download(Uri link) { throw new NotImplementedException(); } - - } } diff --git a/src/Jackett/Indexers/TorrentDay.cs b/src/Jackett/Indexers/TorrentDay.cs index 7399366ec..529c814a6 100644 --- a/src/Jackett/Indexers/TorrentDay.cs +++ b/src/Jackett/Indexers/TorrentDay.cs @@ -2,6 +2,7 @@ using Jackett.Models; using Jackett.Services; using Jackett.Utils; +using Jackett.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using System; @@ -18,49 +19,24 @@ namespace Jackett.Indexers { public class TorrentDay : BaseIndexer, IIndexer { - private readonly string StartPageUrl = ""; - private readonly string LoginUrl = ""; - private readonly string SearchUrl = ""; + private string StartPageUrl { get { return SiteLink + "login.php"; } } + private string LoginUrl { get { return SiteLink + "tak3login.php"; } } + private string SearchUrl { get { return SiteLink + "browse.php?search={0}&cata=yes&c2=1&c7=1&c14=1&c24=1&c26=1&c31=1&c32=1&c33=1"; } } - CookieContainer cookies; - HttpClientHandler handler; - HttpClient client; - - public TorrentDay(IIndexerManagerService i, Logger l) + public TorrentDay(IIndexerManagerService i, Logger l, IWebClient wc) : base(name: "TorrentDay", description: "TorrentDay", - link: new Uri("https://torrentday.eu"), + link: "https://torrentday.eu/", caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), manager: i, + client: wc, logger: l) { - StartPageUrl = SiteLink + "login.php"; - LoginUrl = SiteLink + "tak3login.php"; - SearchUrl = SiteLink + "browse.php?search={0}&cata=yes&c2=1&c7=1&c14=1&c24=1&c26=1&c31=1&c32=1&c33=1"; - - cookies = new CookieContainer(); - handler = new HttpClientHandler - { - CookieContainer = cookies, - AllowAutoRedirect = true, - UseCookies = true, - }; - client = new HttpClient(handler); } public Task GetConfigurationForSetup() { - var config = new ConfigurationDataBasicLogin(); - return Task.FromResult(config); - } - - HttpRequestMessage CreateHttpRequest(string uri) - { - var message = new HttpRequestMessage(); - message.Method = HttpMethod.Get; - message.RequestUri = new Uri(uri); - message.Headers.UserAgent.ParseAdd(BrowserUtil.ChromeUserAgent); - return message; + return Task.FromResult(new ConfigurationDataBasicLogin()); } public async Task ApplyConfiguration(JToken configJson) @@ -68,56 +44,35 @@ namespace Jackett.Indexers var config = new ConfigurationDataBasicLogin(); config.LoadValuesFromJson(configJson); - var startMessage = CreateHttpRequest(StartPageUrl); - var results = await (await client.SendAsync(startMessage)).Content.ReadAsStringAsync(); + var startMessage = await RequestStringWithCookies(StartPageUrl, string.Empty); var pairs = new Dictionary { { "username", config.Username.Value }, { "password", config.Password.Value } }; - var content = new FormUrlEncodedContent(pairs); - var loginRequest = CreateHttpRequest(LoginUrl); - loginRequest.Method = HttpMethod.Post; - loginRequest.Content = content; - loginRequest.Headers.Referrer = new Uri(StartPageUrl); - var response = await client.SendAsync(loginRequest); - var responseContent = await response.Content.ReadAsStringAsync(); - - if (!responseContent.Contains("logout.php")) + var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, null, true, SiteLink, LoginUrl); + ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("logout.php"), () => { - CQ dom = responseContent; + CQ dom = result.Content; var messageEl = dom["#login"]; messageEl.Children("form").Remove(); var errorMessage = messageEl.Text().Trim(); throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)config); - } - else - { - var configSaveData = new JObject(); - cookies.DumpToJson(SiteLink, configSaveData); - SaveConfig(configSaveData); - IsConfigured = true; - } - } - - public void LoadFromSavedConfiguration(JToken jsonConfig) - { - cookies.FillFromJson(SiteLink, jsonConfig, logger); - IsConfigured = true; + }); } public async Task PerformQuery(TorznabQuery query) { - List releases = new List(); - + var releases = new List(); var searchString = query.SanitizedSearchTerm + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = string.Format(SearchUrl, HttpUtility.UrlEncode(searchString)); - var results = await client.GetStringAsync(episodeSearchUrl); + var results = await RequestStringWithCookies(episodeSearchUrl); + try { - CQ dom = results; + CQ dom = results.Content; var rows = dom["#torrentTable > tbody > tr.browse"]; foreach (var row in rows) { @@ -146,14 +101,9 @@ namespace Jackett.Indexers } catch (Exception ex) { - OnParseError(results, ex); + OnParseError(results.Content, ex); } return releases.ToArray(); } - - public Task Download(Uri link) - { - return client.GetByteArrayAsync(link); - } } } diff --git a/src/Jackett/Indexers/TorrentLeech.cs b/src/Jackett/Indexers/TorrentLeech.cs index 52c4bc4bc..131bc3e52 100644 --- a/src/Jackett/Indexers/TorrentLeech.cs +++ b/src/Jackett/Indexers/TorrentLeech.cs @@ -2,6 +2,7 @@ using Jackett.Models; using Jackett.Services; using Jackett.Utils; +using Jackett.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using System; @@ -18,45 +19,29 @@ namespace Jackett.Indexers { public class TorrentLeech : BaseIndexer, IIndexer { - private readonly string LoginUrl = ""; - private readonly string SearchUrl = ""; + private string LoginUrl { get { return SiteLink + "user/account/login/"; } } + private string SearchUrl { get { return SiteLink + "torrents/browse/index/query/{0}/categories/2%2C26%2C27%2C32/orderby/added?"; } } - CookieContainer cookies; - HttpClientHandler handler; - HttpClient client; - - public TorrentLeech(IIndexerManagerService i, Logger l) + public TorrentLeech(IIndexerManagerService i, Logger l, IWebClient wc) : base(name: "TorrentLeech", description: "This is what happens when you seed", - link: new Uri("http://www.torrentleech.org"), + link: "http://www.torrentleech.org/", caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), manager: i, + client: wc, logger: l) { - LoginUrl = SiteLink + "user/account/login/"; - SearchUrl = SiteLink + "torrents/browse/index/query/{0}/categories/2%2C26%2C27%2C32/orderby/added?"; - - cookies = new CookieContainer(); - handler = new HttpClientHandler - { - CookieContainer = cookies, - AllowAutoRedirect = true, - UseCookies = true, - }; - client = new HttpClient(handler); } public Task GetConfigurationForSetup() { - var config = new ConfigurationDataBasicLogin(); - return Task.FromResult(config); + return Task.FromResult(new ConfigurationDataBasicLogin()); } public async Task ApplyConfiguration(JToken configJson) { var config = new ConfigurationDataBasicLogin(); config.LoadValuesFromJson(configJson); - var pairs = new Dictionary { { "username", config.Username.Value }, { "password", config.Password.Value }, @@ -64,31 +49,14 @@ namespace Jackett.Indexers { "login", "submit" } }; - var content = new FormUrlEncodedContent(pairs); - - var response = await client.PostAsync(LoginUrl, content); - var responseContent = await response.Content.ReadAsStringAsync(); - - if (!responseContent.Contains("/user/account/logout")) + var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, null, true,null, LoginUrl); + ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("/user/account/logout"), () => { - CQ dom = responseContent; + CQ dom = result.Content; var messageEl = dom[".ui-state-error"].Last(); var errorMessage = messageEl.Text().Trim(); throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)config); - } - else - { - var configSaveData = new JObject(); - cookies.DumpToJson(SiteLink, configSaveData); - SaveConfig(configSaveData); - IsConfigured = true; - } - } - - public void LoadFromSavedConfiguration(JToken jsonConfig) - { - cookies.FillFromJson(SiteLink, jsonConfig, logger); - IsConfigured = true; + }); } public async Task PerformQuery(TorznabQuery query) @@ -97,10 +65,10 @@ namespace Jackett.Indexers var searchString = query.SanitizedSearchTerm + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = string.Format(SearchUrl, HttpUtility.UrlEncode(searchString)); - var results = await client.GetStringAsync(episodeSearchUrl); + var results = await RequestStringWithCookies(episodeSearchUrl); try { - CQ dom = results; + CQ dom = results.Content; CQ qRows = dom["#torrenttable > tbody > tr"]; @@ -140,18 +108,10 @@ namespace Jackett.Indexers catch (Exception ex) { - OnParseError(results, ex); + OnParseError(results.Content, ex); } return releases.ToArray(); } - - public Task Download(Uri link) - { - return client.GetByteArrayAsync(link); - } - - - } } diff --git a/src/Jackett/Indexers/TorrentShack.cs b/src/Jackett/Indexers/TorrentShack.cs index 97eb3f99a..a01aa47fb 100644 --- a/src/Jackett/Indexers/TorrentShack.cs +++ b/src/Jackett/Indexers/TorrentShack.cs @@ -2,6 +2,7 @@ using Jackett.Models; using Jackett.Services; using Jackett.Utils; +using Jackett.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using System; @@ -18,37 +19,23 @@ namespace Jackett.Indexers { public class TorrentShack : BaseIndexer, IIndexer { - private readonly string LoginUrl = ""; - private readonly string SearchUrl = ""; + private string LoginUrl { get { return SiteLink + "login.php"; } } + private string SearchUrl { get { return SiteLink + "torrents.php?searchstr={0}&release_type=both&searchtags=&tags_type=0&order_by=s3&order_way=desc&torrent_preset=all&filter_cat%5B600%5D=1&filter_cat%5B620%5D=1&filter_cat%5B700%5D=1&filter_cat%5B981%5D=1&filter_cat%5B980%5D=1"; } } - CookieContainer cookies; - HttpClientHandler handler; - HttpClient client; - - public TorrentShack(IIndexerManagerService i, Logger l) + public TorrentShack(IIndexerManagerService i, Logger l, IWebClient wc) : base(name: "TorrentShack", description: "TorrentShack", - link: new Uri("http://torrentshack.me"), + link: "http://torrentshack.me/", caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), + client: wc, manager: i, logger: l) { - LoginUrl = SiteLink + "login.php"; - SearchUrl = SiteLink + "torrents.php?searchstr={0}&release_type=both&searchtags=&tags_type=0&order_by=s3&order_way=desc&torrent_preset=all&filter_cat%5B600%5D=1&filter_cat%5B620%5D=1&filter_cat%5B700%5D=1&filter_cat%5B981%5D=1&filter_cat%5B980%5D=1"; - cookies = new CookieContainer(); - handler = new HttpClientHandler - { - CookieContainer = cookies, - AllowAutoRedirect = true, - UseCookies = true, - }; - client = new HttpClient(handler); } public Task GetConfigurationForSetup() { - var config = new ConfigurationDataBasicLogin(); - return Task.FromResult(config); + return Task.FromResult(new ConfigurationDataBasicLogin()); } public async Task ApplyConfiguration(JToken configJson) @@ -63,46 +50,26 @@ namespace Jackett.Indexers { "login", "Login" } }; - var content = new FormUrlEncodedContent(pairs); - - var response = await client.PostAsync(LoginUrl, content); - var responseContent = await response.Content.ReadAsStringAsync(); - - if (!responseContent.Contains("logout.php")) + var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, null, true, null, LoginUrl); + ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("logout.php"), () => { - CQ dom = responseContent; + CQ dom = result.Content; var messageEl = dom["#loginform"]; messageEl.Children("table").Remove(); var errorMessage = messageEl.Text().Trim(); throw new ExceptionWithConfigData(errorMessage, (ConfigurationData)config); - } - else - { - var configSaveData = new JObject(); - cookies.DumpToJson(SiteLink, configSaveData); - SaveConfig(configSaveData); - IsConfigured = true; - } - - } - - public void LoadFromSavedConfiguration(JToken jsonConfig) - { - cookies.FillFromJson(SiteLink, jsonConfig, logger); - IsConfigured = true; + }); } public async Task PerformQuery(TorznabQuery query) { - List releases = new List(); - - + var releases = new List(); var searchString = query.SanitizedSearchTerm + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = string.Format(SearchUrl, HttpUtility.UrlEncode(searchString)); - var results = await client.GetStringAsync(episodeSearchUrl); + var results = await RequestStringWithCookies(episodeSearchUrl); try { - CQ dom = results; + CQ dom = results.Content; var rows = dom["#torrent_table > tbody > tr.torrent"]; foreach (var row in rows) { @@ -130,14 +97,9 @@ namespace Jackett.Indexers } catch (Exception ex) { - OnParseError(results, ex); + OnParseError(results.Content, ex); } return releases.ToArray(); } - - public Task Download(Uri link) - { - return client.GetByteArrayAsync(link); - } } } diff --git a/src/Jackett/Indexers/Torrentz.cs b/src/Jackett/Indexers/Torrentz.cs index b0246d82d..fe069fce6 100644 --- a/src/Jackett/Indexers/Torrentz.cs +++ b/src/Jackett/Indexers/Torrentz.cs @@ -1,6 +1,7 @@ using Jackett.Models; using Jackett.Services; using Jackett.Utils; +using Jackett.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using System; @@ -17,38 +18,23 @@ namespace Jackett.Indexers { public class Torrentz : BaseIndexer, IIndexer { - private readonly string SearchUrl = ""; - string BaseUrl; + private string SearchUrl { get { return SiteLink + "feed_verifiedP?f={0}"; } } + private string BaseUrl; - CookieContainer cookies; - HttpClientHandler handler; - HttpClient client; - - public Torrentz(IIndexerManagerService i, Logger l) + public Torrentz(IIndexerManagerService i, Logger l, IWebClient wc) : base(name: "Torrentz", description: "Torrentz is a meta-search engine and a Multisearch. This means we just search other search engines.", - link: new Uri("https://torrentz.eu"), + link: "https://torrentz.eu/", caps: TorznabCapsUtil.CreateDefaultTorznabTVCaps(), manager: i, + client: wc, logger: l) { - - SearchUrl = SiteLink + "feed_verifiedP?f={0}"; - cookies = new CookieContainer(); - handler = new HttpClientHandler - { - CookieContainer = cookies, - AllowAutoRedirect = true, - UseCookies = true, - }; - client = new HttpClient(handler); } public Task GetConfigurationForSetup() { - var config = new ConfigurationDataUrl(SiteLink); - return Task.FromResult(config); - + return Task.FromResult(new ConfigurationDataUrl(SiteLink)); } public async Task ApplyConfiguration(JToken configJson) @@ -62,7 +48,6 @@ namespace Jackett.Indexers throw new Exception("Could not find releases from this URL"); BaseUrl = formattedUrl; - var configSaveData = new JObject(); configSaveData["base_url"] = BaseUrl; SaveConfig(configSaveData); @@ -70,33 +55,18 @@ namespace Jackett.Indexers } - private WebClient getWebClient() - { - WebClient wc = new WebClient(); - WebHeaderCollection headers = new WebHeaderCollection(); - headers.Add("User-Agent", BrowserUtil.ChromeUserAgent); - wc.Headers = headers; - return wc; - } - async Task PerformQuery(TorznabQuery query, string baseUrl) { - List releases = new List(); - + var releases = new List(); var searchString = query.SanitizedSearchTerm + " " + query.GetEpisodeSearchString(); var episodeSearchUrl = string.Format(SearchUrl, HttpUtility.UrlEncode(searchString.Trim())); - - XmlDocument xmlDoc = new XmlDocument(); + var xmlDoc = new XmlDocument(); string xml = string.Empty; - WebClient wc = getWebClient(); + var result = await RequestStringWithCookies(episodeSearchUrl); try { - using (wc) - { - xml = await wc.DownloadStringTaskAsync(new Uri(episodeSearchUrl)); - xmlDoc.LoadXml(xml); - } + xmlDoc.LoadXml(result.Content); ReleaseInfo release; TorrentzHelper td; @@ -135,7 +105,7 @@ namespace Jackett.Indexers } - public void LoadFromSavedConfiguration(JToken jsonConfig) + public override void LoadFromSavedConfiguration(JToken jsonConfig) { BaseUrl = (string)jsonConfig["base_url"]; IsConfigured = true; @@ -146,7 +116,7 @@ namespace Jackett.Indexers return await PerformQuery(query, BaseUrl); } - public Task Download(Uri link) + public override Task Download(Uri link) { throw new NotImplementedException(); } diff --git a/src/Jackett/Jackett.csproj b/src/Jackett/Jackett.csproj index 9191460f1..15c99e613 100644 --- a/src/Jackett/Jackett.csproj +++ b/src/Jackett/Jackett.csproj @@ -163,20 +163,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -198,27 +224,7 @@ - - - - - - - - - - - - - - - - - - - - @@ -239,7 +245,6 @@ - diff --git a/src/Jackett/JackettModule.cs b/src/Jackett/JackettModule.cs index 23cfae951..3d48c342d 100644 --- a/src/Jackett/JackettModule.cs +++ b/src/Jackett/JackettModule.cs @@ -8,6 +8,7 @@ using Autofac.Integration.WebApi; using Jackett.Indexers; using Jackett.Utils; using Jackett.Utils.Clients; +using AutoMapper; namespace Jackett { @@ -41,6 +42,14 @@ namespace Jackett { builder.RegisterType(indexer).Named(BaseIndexer.GetIndexerID(indexer)); } + + Mapper.CreateMap().AfterMap((be, str) => + { + str.Content = Encoding.UTF8.GetString(be.Content); + }); + + Mapper.CreateMap(); + Mapper.CreateMap(); } } } diff --git a/src/Jackett/Models/ConfigurationData.cs b/src/Jackett/Models/ConfigurationData.cs index 21344cf9c..08ff9b105 100644 --- a/src/Jackett/Models/ConfigurationData.cs +++ b/src/Jackett/Models/ConfigurationData.cs @@ -19,6 +19,16 @@ namespace Jackett.Models HiddenData } + public ConfigurationData() + { + + } + + public ConfigurationData(JToken json) + { + LoadValuesFromJson(json); + } + public void LoadValuesFromJson(JToken json) { // todo: match up ids with items and fill values diff --git a/src/Jackett/Models/ConfigurationDataBasicLogin.cs b/src/Jackett/Models/ConfigurationDataBasicLogin.cs index 66afbffe3..fc7550284 100644 --- a/src/Jackett/Models/ConfigurationDataBasicLogin.cs +++ b/src/Jackett/Models/ConfigurationDataBasicLogin.cs @@ -1,4 +1,5 @@ -using System; +using Newtonsoft.Json.Linq; +using System; using System.Collections.Generic; using System.Linq; using System.Text; diff --git a/src/Jackett/Models/IndexerConfig/BmtvConfig.cs b/src/Jackett/Models/IndexerConfig/BmtvConfig.cs new file mode 100644 index 000000000..c3c431e24 --- /dev/null +++ b/src/Jackett/Models/IndexerConfig/BmtvConfig.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Jackett.Models.IndexerConfig +{ + class BmtvConfig : ConfigurationData + { + public StringItem Username { get; private set; } + + public StringItem Password { get; private set; } + + public ImageItem CaptchaImage { get; private set; } + + public StringItem CaptchaText { get; private set; } + + public BmtvConfig() + { + Username = new StringItem { Name = "Username" }; + Password = new StringItem { Name = "Password" }; + CaptchaImage = new ImageItem { Name = "Captcha Image" }; + CaptchaText = new StringItem { Name = "Captcha Text" }; + } + + public override Item[] GetItems() + { + return new Item[] { Username, Password, CaptchaImage, CaptchaText }; + } + } +} diff --git a/src/Jackett/Models/IndexerConfig/ConfigurationDataBasicLoginAnimeBytes.cs b/src/Jackett/Models/IndexerConfig/ConfigurationDataBasicLoginAnimeBytes.cs new file mode 100644 index 000000000..249bec8d7 --- /dev/null +++ b/src/Jackett/Models/IndexerConfig/ConfigurationDataBasicLoginAnimeBytes.cs @@ -0,0 +1,27 @@ +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Jackett.Models.IndexerConfig +{ + class ConfigurationDataBasicLoginAnimeBytes : ConfigurationDataBasicLogin + { + public BoolItem IncludeRaw { get; private set; } + public DisplayItem DateWarning { get; private set; } + + public ConfigurationDataBasicLoginAnimeBytes() + : base() + { + IncludeRaw = new BoolItem() { Name = "IncludeRaw", Value = false }; + DateWarning = new DisplayItem("This tracker does not supply upload dates so they are based off year of release.") { Name = "DateWarning" }; + } + + public override Item[] GetItems() + { + return new Item[] { Username, Password, IncludeRaw, DateWarning }; + } + } +} diff --git a/src/Jackett/Models/IndexerConfig/ConfigurationDataBasicLoginFrenchTorrentDb.cs b/src/Jackett/Models/IndexerConfig/ConfigurationDataBasicLoginFrenchTorrentDb.cs new file mode 100644 index 000000000..9cfbb6d6d --- /dev/null +++ b/src/Jackett/Models/IndexerConfig/ConfigurationDataBasicLoginFrenchTorrentDb.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Jackett.Models.IndexerConfig +{ + class ConfigurationDataBasicLoginFrenchTorrentDb : ConfigurationData + { + public StringItem Cookie { get; private set; } + + public ConfigurationDataBasicLoginFrenchTorrentDb() + { + Cookie = new StringItem { Name = "Cookie" }; + } + + public override Item[] GetItems() + { + return new Item[] { Cookie }; + } + } +} diff --git a/src/Jackett/Models/IndexerConfig/PretomeConfiguration.cs b/src/Jackett/Models/IndexerConfig/PretomeConfiguration.cs new file mode 100644 index 000000000..27cb36688 --- /dev/null +++ b/src/Jackett/Models/IndexerConfig/PretomeConfiguration.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Jackett.Models.IndexerConfig +{ + class PretomeConfiguration : ConfigurationDataBasicLogin + { + public StringItem Pin { get; private set; } + + public PretomeConfiguration() : base() + { + Pin = new StringItem { Name = "Login Pin Number" }; + } + + public override Item[] GetItems() + { + return new Item[] { Pin, Username, Password }; + } + } +} diff --git a/src/Jackett/Models/TorznabCapabilities.cs b/src/Jackett/Models/TorznabCapabilities.cs index 012ad8e57..d40fff56c 100644 --- a/src/Jackett/Models/TorznabCapabilities.cs +++ b/src/Jackett/Models/TorznabCapabilities.cs @@ -23,6 +23,12 @@ namespace Jackett.Models Categories = new List(); } + public TorznabCapabilities(params TorznabCategory[] cats) + { + Categories = new List(); + Categories.AddRange(cats); + } + string SupportedTVSearchParams { get diff --git a/src/Jackett/Models/TorznabCategory.cs b/src/Jackett/Models/TorznabCategory.cs index 53425c540..f1e059aa8 100644 --- a/src/Jackett/Models/TorznabCategory.cs +++ b/src/Jackett/Models/TorznabCategory.cs @@ -17,5 +17,53 @@ namespace Jackett.Models { SubCategories = new List(); } + + public static TorznabCategory Anime + { + get + { + return new TorznabCategory() + { + ID = "5070", + Name = "TV/Anime" + }; + } + } + + public static TorznabCategory TV + { + get + { + return new TorznabCategory() + { + ID = "5000", + Name = "TV" + }; + } + } + + public static TorznabCategory TVSD + { + get + { + return new TorznabCategory() + { + ID = "5030", + Name = "TV/SD" + }; + } + } + + public static TorznabCategory TVHD + { + get + { + return new TorznabCategory() + { + ID = "5040", + Name = "TV/HD" + }; + } + } } } diff --git a/src/Jackett/Properties/AssemblyInfo.cs b/src/Jackett/Properties/AssemblyInfo.cs index c854b4d9e..bd946bcd9 100644 --- a/src/Jackett/Properties/AssemblyInfo.cs +++ b/src/Jackett/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ using System.Runtime.InteropServices; // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("0.5.1.0")] +[assembly: AssemblyVersion("0.6.0.0")] [assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/src/Jackett/Utils/Clients/BaseWebResult.cs b/src/Jackett/Utils/Clients/BaseWebResult.cs new file mode 100644 index 000000000..2f2a4e9d2 --- /dev/null +++ b/src/Jackett/Utils/Clients/BaseWebResult.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Text; +using System.Threading.Tasks; + +namespace Jackett.Utils.Clients +{ + public abstract class BaseWebResult + { + public HttpStatusCode Status { get; set; } + public string Cookies { get; set; } + public string RedirectingTo { get; set; } + } +} diff --git a/src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs b/src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs index db0405788..b81e9a54c 100644 --- a/src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs +++ b/src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs @@ -1,4 +1,5 @@ -using Jackett.Models; +using AutoMapper; +using Jackett.Models; using Jackett.Services; using NLog; using System; @@ -23,10 +24,23 @@ namespace Jackett.Utils.Clients public async Task GetBytes(WebRequest request) { - Jackett.CurlHelper.CurlResponse response; - logger.Debug(string.Format("UnixLibCurlWebClient:GetBytes(Url:{0})", request.Url)); + var result = await Run(request); + logger.Debug(string.Format("UnixLibCurlWebClient: Returning", result.Status)); + return result; + } + public async Task GetString(WebRequest request) + { + logger.Debug(string.Format("UnixLibCurlWebClient:GetString(Url:{0})", request.Url)); + var result = await Run(request); + logger.Debug(string.Format("UnixLibCurlWebClient: Returning", result.Status)); + return Mapper.Map(result); + } + + private async Task Run(WebRequest request) + { + Jackett.CurlHelper.CurlResponse response; if (request.Type == RequestType.GET) { response = await CurlHelper.GetAsync(request.Url, request.Cookies, request.Referer); @@ -45,32 +59,17 @@ namespace Jackett.Utils.Clients if (response.Headers != null) { - foreach(var header in response.Headers) + foreach (var header in response.Headers) { - if(string.Equals(header.Key, "location", StringComparison.InvariantCultureIgnoreCase) && header.Value !=null) + if (string.Equals(header.Key, "location", StringComparison.InvariantCultureIgnoreCase) && header.Value != null) { result.RedirectingTo = header.Value; } } } - logger.Debug(string.Format("UnixLibCurlWebClient: Returning", result.Status)); + ServerUtil.ResureRedirectIsFullyQualified(request, result); return result; } - - public async Task GetString(WebRequest request) - { - logger.Debug(string.Format("UnixLibCurlWebClient:GetString(Url:{0})", request.Url)); - var result = await GetBytes(request); - - var sresult = new WebClientStringResult() - { - Content = Encoding.UTF8.GetString(result.Content), - Cookies = result.Cookies, - Status = result.Status - }; - - return sresult; - } } } diff --git a/src/Jackett/Utils/Clients/UnixSafeCurlWebClient.cs b/src/Jackett/Utils/Clients/UnixSafeCurlWebClient.cs index f08229508..623afda59 100644 --- a/src/Jackett/Utils/Clients/UnixSafeCurlWebClient.cs +++ b/src/Jackett/Utils/Clients/UnixSafeCurlWebClient.cs @@ -1,4 +1,5 @@ -using Jackett.Models; +using AutoMapper; +using Jackett.Models; using Jackett.Services; using NLog; using System; @@ -23,30 +24,26 @@ namespace Jackett.Utils.Clients logger = l; } - public Task GetBytes(WebRequest request) + public async Task GetBytes(WebRequest request) { logger.Debug(string.Format("UnixSafeCurlWebClient:GetBytes(Url:{0})", request.Url)); - return Run(request); + var result = await Run(request); + logger.Debug(string.Format("UnixSafeCurlWebClient: Returning", result.Status)); + return result; } public async Task GetString(WebRequest request) { logger.Debug(string.Format("UnixSafeCurlWebClient:GetString(Url:{0})", request.Url)); - var byteResult = await Run(request); - return new WebClientStringResult() - { - Cookies = byteResult.Cookies, - Status = byteResult.Status, - Content = Encoding.UTF8.GetString(byteResult.Content), - RedirectingTo = byteResult.RedirectingTo - }; + var result = await Run(request); + logger.Debug(string.Format("UnixSafeCurlWebClient: Returning", result.Status)); + return Mapper.Map(result); } private async Task Run(WebRequest request) { var args = new StringBuilder(); args.AppendFormat("--url \"{0}\" ", request.Url); - args.AppendFormat("-i -sS --user-agent \"{0}\" ", BrowserUtil.ChromeUserAgent); if (!string.IsNullOrWhiteSpace(request.Cookies)) @@ -66,7 +63,6 @@ namespace Jackett.Utils.Clients } var tempFile = Path.GetTempFileName(); - args.AppendFormat("--output \"{0}\" ", tempFile); string stdout = null; @@ -77,9 +73,7 @@ namespace Jackett.Utils.Clients var outputData = File.ReadAllBytes(tempFile); File.Delete(tempFile); - stdout = Encoding.UTF8.GetString(outputData); - var result = new WebClientByteResult(); var headSplit = stdout.IndexOf("\r\n\r\n"); if (headSplit < 0) @@ -127,6 +121,7 @@ namespace Jackett.Utils.Clients } logger.Debug("WebClientByteResult returned " + result.Status); + ServerUtil.ResureRedirectIsFullyQualified(request, result); return result; } } diff --git a/src/Jackett/Utils/Clients/WebByteResult.cs b/src/Jackett/Utils/Clients/WebByteResult.cs index b904ec070..f0ae9de90 100644 --- a/src/Jackett/Utils/Clients/WebByteResult.cs +++ b/src/Jackett/Utils/Clients/WebByteResult.cs @@ -7,11 +7,8 @@ using System.Threading.Tasks; namespace Jackett.Utils.Clients { - public class WebClientByteResult + public class WebClientByteResult : BaseWebResult { - public HttpStatusCode Status { get; set; } - public string Cookies { get; set; } public byte[] Content { get; set; } - public string RedirectingTo { get; set; } } } diff --git a/src/Jackett/Utils/Clients/WebClientResult.cs b/src/Jackett/Utils/Clients/WebClientResult.cs index a801e8800..2898e05eb 100644 --- a/src/Jackett/Utils/Clients/WebClientResult.cs +++ b/src/Jackett/Utils/Clients/WebClientResult.cs @@ -7,11 +7,8 @@ using System.Threading.Tasks; namespace Jackett.Utils.Clients { - public class WebClientStringResult + public class WebClientStringResult: BaseWebResult { - public HttpStatusCode Status { get; set; } - public string Cookies { get; set; } public string Content { get; set; } - public string RedirectingTo { get; set; } } } diff --git a/src/Jackett/Utils/Clients/WindowsWebClient.cs b/src/Jackett/Utils/Clients/WindowsWebClient.cs index 12048fe05..747d7755e 100644 --- a/src/Jackett/Utils/Clients/WindowsWebClient.cs +++ b/src/Jackett/Utils/Clients/WindowsWebClient.cs @@ -1,4 +1,5 @@ -using Jackett.Models; +using AutoMapper; +using Jackett.Models; using NLog; using System; using System.Collections.Generic; @@ -13,7 +14,6 @@ namespace Jackett.Utils.Clients class WindowsWebClient : IWebClient { private Logger logger; - public WindowsWebClient(Logger l) { @@ -24,7 +24,21 @@ namespace Jackett.Utils.Clients public async Task GetBytes(WebRequest request) { logger.Debug(string.Format("WindowsWebClient:GetBytes(Url:{0})", request.Url)); + var result = await Run(request); + logger.Debug(string.Format("WindowsWebClient: Returning", result.Status)); + return result; + } + public async Task GetString(WebRequest request) + { + logger.Debug(string.Format("WindowsWebClient:GetString(Url:{0})", request.Url)); + var result = await Run(request); + logger.Debug(string.Format("WindowsWebClient: Returning", result.Status)); + return Mapper.Map(result); + } + + private async Task Run(WebRequest request) + { var cookies = new CookieContainer(); if (!string.IsNullOrEmpty(request.Cookies)) { @@ -64,7 +78,10 @@ namespace Jackett.Utils.Clients var result = new WebClientByteResult(); result.Content = await response.Content.ReadAsByteArrayAsync(); - + if (response.Headers.Location != null) + { + result.RedirectingTo = response.Headers.Location.ToString(); + } result.Status = response.StatusCode; // Compatiblity issue between the cookie format and httpclient @@ -76,83 +93,13 @@ namespace Jackett.Utils.Clients var cookieBuilder = new StringBuilder(); foreach (var c in cookieHeaders) { - cookieBuilder.AppendFormat("{0} ", c.Substring(0, c.LastIndexOf(';'))); + cookieBuilder.AppendFormat("{0} ", c.Substring(0, c.IndexOf(';')+1)); } result.Cookies = cookieBuilder.ToString().TrimEnd(); } - return result; - } - - public async Task GetString(WebRequest request) - { - logger.Debug(string.Format("WindowsWebClient:GetString(Url:{0})", request.Url)); - var cookies = new CookieContainer(); - - if (!string.IsNullOrEmpty(request.Cookies)) - { - var uri = new Uri(request.Url); - foreach (var c in request.Cookies.Split(';')) - { - try - { - cookies.SetCookies(uri, c); - } - catch (CookieException ex) - { - logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message); - } - } - } - - var client = new HttpClient(new HttpClientHandler - { - CookieContainer = cookies, - AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more. - UseCookies = true, - }); - - client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent); - HttpResponseMessage response = null; - - if (request.Type == RequestType.POST) - { - var content = new FormUrlEncodedContent(request.PostData); - response = await client.PostAsync(request.Url, content); - } else - { - response = await client.GetAsync(request.Url); - } - - var result = new WebClientStringResult(); - result.Content = await response.Content.ReadAsStringAsync(); - - // Compatiblity issue between the cookie format and httpclient - // Pull it out manually ignoring the expiry date then set it manually - // http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer - IEnumerable cookieHeaders; - if (response.Headers.TryGetValues("set-cookie", out cookieHeaders)) - { - var cookieBuilder = new StringBuilder(); - foreach (var c in cookieHeaders) - { - if (cookieBuilder.Length > 0) - { - cookieBuilder.Append("; "); - } - - cookieBuilder.Append( c.Substring(0, c.IndexOf(';'))); - } - - result.Cookies = cookieBuilder.ToString(); - } - - result.Status = response.StatusCode; - if (null != response.Headers.Location) - { - result.RedirectingTo = response.Headers.Location.ToString(); - } + ServerUtil.ResureRedirectIsFullyQualified(request, result); return result; } } diff --git a/src/Jackett/Utils/ServerUtil.cs b/src/Jackett/Utils/ServerUtil.cs index 6e3f16d32..a176e810f 100644 --- a/src/Jackett/Utils/ServerUtil.cs +++ b/src/Jackett/Utils/ServerUtil.cs @@ -1,4 +1,5 @@ -using System; +using Jackett.Utils.Clients; +using System; using System.Collections.Generic; using System.Linq; using System.Security.Principal; @@ -98,5 +99,19 @@ namespace Jackett.Utils } return isAdmin; } + + public static void ResureRedirectIsFullyQualified(WebRequest req, BaseWebResult result) + { + if (!string.IsNullOrEmpty(result.RedirectingTo)) + { + var destLower = result.RedirectingTo.ToLowerInvariant(); + if (!destLower.StartsWith("http")) + { + var hostUri = new Uri(req.Url); + var fullUri = new Uri(hostUri, result.RedirectingTo); + result.RedirectingTo = fullUri.ToString(); + } + } + } } } diff --git a/src/Jackett/Utils/StringUtil.cs b/src/Jackett/Utils/StringUtil.cs index 6a40ca5f9..499a5ee08 100644 --- a/src/Jackett/Utils/StringUtil.cs +++ b/src/Jackett/Utils/StringUtil.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Security.Cryptography; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; @@ -21,5 +22,21 @@ namespace Jackett.Utils return Encoding.UTF8.GetString(Convert.FromBase64String(str)); } + public static string Hash(string input) + { + // Use input string to calculate MD5 hash + MD5 md5 = System.Security.Cryptography.MD5.Create(); + byte[] inputBytes = System.Text.Encoding.ASCII.GetBytes(input); + byte[] hashBytes = md5.ComputeHash(inputBytes); + + // Convert the byte array to hexadecimal string + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < hashBytes.Length; i++) + { + sb.Append(hashBytes[i].ToString("X2")); + } + return sb.ToString(); + } + } } diff --git a/src/Jackett/Utils/TorznabCapsUtil.cs b/src/Jackett/Utils/TorznabCapsUtil.cs index 789d38236..d9a25b23e 100644 --- a/src/Jackett/Utils/TorznabCapsUtil.cs +++ b/src/Jackett/Utils/TorznabCapsUtil.cs @@ -15,10 +15,10 @@ namespace Jackett.Utils caps.SearchAvailable = true; caps.TVSearchAvailable = true; caps.SupportsTVRageSearch = false; - caps.Categories.AddRange(new[] { - new TorznabCategory { ID = "5000", Name = "TV" }, - new TorznabCategory { ID = "5030", Name = "TV/SD" }, - new TorznabCategory { ID = "5040", Name = "TV/HD" } + caps.Categories.AddRange(new[] { + TorznabCategory.TV, + TorznabCategory.TVSD, + TorznabCategory.TVHD }); return caps; }