From d499c291b9ea54b7d54fd1cbb648c47b1ab7fd82 Mon Sep 17 00:00:00 2001 From: kaso17 Date: Tue, 6 Dec 2016 14:56:47 +0100 Subject: [PATCH] WebRequest: make encoding configurable (#816) --- src/Jackett.Test/TestWebClient.cs | 17 +++- src/Jackett/Definitions/alphareign.yml | 2 +- src/Jackett/Definitions/apollo.yml | 1 + src/Jackett/Definitions/blubits.yml | 1 + src/Jackett/Definitions/eotforum.yml | 1 + src/Jackett/Definitions/ethor.yml | 1 + src/Jackett/Definitions/funfile.yml | 80 +++++++++++++++++ src/Jackett/Definitions/hdme.yml | 1 + src/Jackett/Definitions/newretro.yml | 1 + src/Jackett/Definitions/sdbits.yml | 1 + src/Jackett/Definitions/shareisland.yml | 1 + src/Jackett/Definitions/thehorrorcharnel.yml | 1 + src/Jackett/Definitions/torrentsectorcrew.yml | 1 + src/Jackett/Definitions/tspate.yml | 1 + src/Jackett/Definitions/uhdbits.yml | 1 + src/Jackett/Definitions/worldofp2p.yml | 1 + .../Indexers/Abstract/AvistazTracker.cs | 2 + src/Jackett/Indexers/AlphaRatio.cs | 2 + src/Jackett/Indexers/Andraste.cs | 6 +- src/Jackett/Indexers/AnimeBytes.cs | 2 +- src/Jackett/Indexers/AnimeTorrents.cs | 2 + src/Jackett/Indexers/BB.cs | 2 + src/Jackett/Indexers/BJShare.cs | 5 +- src/Jackett/Indexers/BaseIndexer.cs | 16 ++-- src/Jackett/Indexers/BestFriends.cs | 3 + src/Jackett/Indexers/BeyondHD.cs | 3 + src/Jackett/Indexers/BitCityReloaded.cs | 3 + src/Jackett/Indexers/BitMeTV.cs | 1 + src/Jackett/Indexers/CardigannIndexer.cs | 11 +-- src/Jackett/Indexers/DanishBits.cs | 3 + src/Jackett/Indexers/Demonoid.cs | 2 + src/Jackett/Indexers/DigitalHive.cs | 3 + src/Jackett/Indexers/FileList.cs | 2 + src/Jackett/Indexers/Freshon.cs | 1 + src/Jackett/Indexers/FunFile.cs | 3 + src/Jackett/Indexers/Fuzer.cs | 8 +- src/Jackett/Indexers/GhostCity.cs | 3 + src/Jackett/Indexers/HD4Free.cs | 3 + src/Jackett/Indexers/HDSpace.cs | 2 + src/Jackett/Indexers/HDTorrents.cs | 2 + src/Jackett/Indexers/Hebits.cs | 8 +- src/Jackett/Indexers/Hounddawgs.cs | 4 +- src/Jackett/Indexers/HouseOfTorrents.cs | 3 + src/Jackett/Indexers/IPTorrents.cs | 2 + src/Jackett/Indexers/ImmortalSeed.cs | 2 + src/Jackett/Indexers/MoreThanTV.cs | 2 + src/Jackett/Indexers/MyAnonamouse.cs | 2 + src/Jackett/Indexers/NewRealWorld.cs | 7 +- src/Jackett/Indexers/PirateTheNet.cs | 3 + src/Jackett/Indexers/Pretome.cs | 1 + src/Jackett/Indexers/SceneAccess.cs | 1 + src/Jackett/Indexers/SceneTime.cs | 3 + src/Jackett/Indexers/TVChaosUK.cs | 2 + src/Jackett/Indexers/TVVault.cs | 3 + src/Jackett/Indexers/TehConnection.cs | 3 + src/Jackett/Indexers/TorrentBytes.cs | 1 + src/Jackett/Indexers/TorrentDay.cs | 2 + src/Jackett/Indexers/TorrentHeaven.cs | 6 +- src/Jackett/Indexers/TorrentLeech.cs | 1 + src/Jackett/Indexers/TorrentNetwork.cs | 6 +- src/Jackett/Indexers/TorrentShack.cs | 2 + src/Jackett/Indexers/TorrentSyndikat.cs | 3 + src/Jackett/Indexers/TransmitheNet.cs | 2 + src/Jackett/Indexers/XSpeeds.cs | 3 + src/Jackett/Indexers/myAmity.cs | 6 +- src/Jackett/Indexers/x264.cs | 3 + src/Jackett/Utils/Clients/HttpWebClient.cs | 41 ++------- src/Jackett/Utils/Clients/IWebClient.cs | 87 +++++++++++++++++-- .../Utils/Clients/UnixLibCurlWebClient.cs | 35 ++------ .../Utils/Clients/UnixSafeCurlWebClient.cs | 30 ++----- src/Jackett/Utils/Clients/WebRequest.cs | 4 + 71 files changed, 351 insertions(+), 129 deletions(-) create mode 100644 src/Jackett/Definitions/funfile.yml diff --git a/src/Jackett.Test/TestWebClient.cs b/src/Jackett.Test/TestWebClient.cs index 4104972ed..33e1bee07 100644 --- a/src/Jackett.Test/TestWebClient.cs +++ b/src/Jackett.Test/TestWebClient.cs @@ -1,4 +1,6 @@ -using Jackett.Utils.Clients; +using Jackett.Services; +using Jackett.Utils.Clients; +using NLog; using System; using System.Collections.Generic; using System.Linq; @@ -12,6 +14,13 @@ namespace JackettTest private Dictionary> byteCallbacks = new Dictionary>(); private Dictionary> stringCallbacks = new Dictionary>(); + public TestWebClient(IProcessService p, Logger l, IConfigurationService c) + : base(p: p, + l: l, + c: c) + { + } + public void RegisterByteCall(WebRequest req, Func f) { byteCallbacks.Add(req, f); @@ -22,17 +31,17 @@ namespace JackettTest stringCallbacks.Add(req, f); } - public Task GetBytes(WebRequest request) + override public Task GetBytes(WebRequest request) { return Task.FromResult< WebClientByteResult>(byteCallbacks.Where(r => r.Key.Equals(request)).First().Value.Invoke(request)); } - public Task GetString(WebRequest request) + override public Task GetString(WebRequest request) { return Task.FromResult(stringCallbacks.Where(r => r.Key.Equals(request)).First().Value.Invoke(request)); } - public void Init() + override public void Init() { } diff --git a/src/Jackett/Definitions/alphareign.yml b/src/Jackett/Definitions/alphareign.yml index 5f9f9d424..aff9db94e 100644 --- a/src/Jackett/Definitions/alphareign.yml +++ b/src/Jackett/Definitions/alphareign.yml @@ -3,7 +3,7 @@ name: AlphaReign description: "A DHT only tracker" language: en-us - + encoding: UTF-8 links: - https://alphareign.se diff --git a/src/Jackett/Definitions/apollo.yml b/src/Jackett/Definitions/apollo.yml index b4b446e4d..99a7b1249 100644 --- a/src/Jackett/Definitions/apollo.yml +++ b/src/Jackett/Definitions/apollo.yml @@ -3,6 +3,7 @@ name: Apollo description: "A music tracker" language: en-us + encoding: UTF-8 links: - https://apollo.rip diff --git a/src/Jackett/Definitions/blubits.yml b/src/Jackett/Definitions/blubits.yml index c4af15135..40d11e9c7 100644 --- a/src/Jackett/Definitions/blubits.yml +++ b/src/Jackett/Definitions/blubits.yml @@ -3,6 +3,7 @@ name: Blu-bits description: "A HD tracker" language: en-us + encoding: UTF-8 links: - https://blu-bits.com/ diff --git a/src/Jackett/Definitions/eotforum.yml b/src/Jackett/Definitions/eotforum.yml index 9802c11a9..8deface6d 100644 --- a/src/Jackett/Definitions/eotforum.yml +++ b/src/Jackett/Definitions/eotforum.yml @@ -3,6 +3,7 @@ name: EoT-Forum description: "A German gerneral tracker" language: de-de + encoding: windows-1252 links: - http://eot-forum.net diff --git a/src/Jackett/Definitions/ethor.yml b/src/Jackett/Definitions/ethor.yml index cc884a158..4e13ad74a 100644 --- a/src/Jackett/Definitions/ethor.yml +++ b/src/Jackett/Definitions/ethor.yml @@ -3,6 +3,7 @@ name: Ethor.net (Thor's Land) description: "A French gerneral tracker" language: fr-fr + encoding: windows-1252 links: - https://ethor.net/ diff --git a/src/Jackett/Definitions/funfile.yml b/src/Jackett/Definitions/funfile.yml new file mode 100644 index 000000000..63fd21623 --- /dev/null +++ b/src/Jackett/Definitions/funfile.yml @@ -0,0 +1,80 @@ +--- + site: funfile + name: FunFile + description: "A general tracker" + language: en-us + encoding: windows-1252 + links: + - https://www.funfile.org/ + + caps: + categories: + 44: TV/Anime # Anime + 22: PC # Applications + 43: Audio/Audiobook # Audio Books + 27: Books # Ebook + 4: PC/Games # Games + 40: Other/Misc # Miscellaneous + 19: Movies # Movies + 6: Audio # Music + 31: PC/Phone-Other # Portable + 7: TV # TV + + modes: + search: [q] + tv-search: [q, season, ep] + + login: + path: /takelogin.php + method: post + form: form + inputs: + username: "{{ .Config.username }}" + password: "{{ .Config.password }}" + login: "Login" + error: + - selector: mf_content + test: + path: /my.php + + ratio: + path: /browse.php + selector: div:has(div#clock) > b + b+ b > span + + search: + path: /browse.php + inputs: + $raw: "{{range .Categories}}c{{.}}=1&{{end}}" + search: "{{ .Query.Keywords }}" + incldead: 1 + showspam: 1 + + rows: + selector: table[cellpadding="2"] > tbody > tr:has(td.row3) + fields: + title: + selector: a[title][href^="details.php"] + attribute: title + category: + selector: a[href^="browse.php?cat="] + attribute: href + filters: + - name: querystring + args: cat + comments: + selector: a[href^="details.php?id="] + attribute: href + download: + selector: a[href^="download.php"] + attribute: href + size: + selector: td:nth-child(8) + seeders: + selector: td:nth-child(10) + leechers: + selector: td:nth-child(11) + date: + selector: td:nth-child(6) + filters: + - name: append + args: " ago" diff --git a/src/Jackett/Definitions/hdme.yml b/src/Jackett/Definitions/hdme.yml index aed290bcd..402cb996e 100644 --- a/src/Jackett/Definitions/hdme.yml +++ b/src/Jackett/Definitions/hdme.yml @@ -2,6 +2,7 @@ site: hdme name: HDME language: en-us + encoding: windows-1252 links: - https://hdme.eu diff --git a/src/Jackett/Definitions/newretro.yml b/src/Jackett/Definitions/newretro.yml index d7ff31bfd..c4fcd2ef7 100644 --- a/src/Jackett/Definitions/newretro.yml +++ b/src/Jackett/Definitions/newretro.yml @@ -3,6 +3,7 @@ name: The New Retro description: "A German gerneral tracker" language: de-de + encoding: windows-1252 links: - http://new-retro.ddns.net/ diff --git a/src/Jackett/Definitions/sdbits.yml b/src/Jackett/Definitions/sdbits.yml index ad0fb7e8b..34374d5b8 100644 --- a/src/Jackett/Definitions/sdbits.yml +++ b/src/Jackett/Definitions/sdbits.yml @@ -3,6 +3,7 @@ name: SDBits description: "SDBits is a small tracker that focuses on SD movies and tv." language: en-us + encoding: UTF-8 links: - http://sdbits.org diff --git a/src/Jackett/Definitions/shareisland.yml b/src/Jackett/Definitions/shareisland.yml index 5073eed7a..66fd703d7 100644 --- a/src/Jackett/Definitions/shareisland.yml +++ b/src/Jackett/Definitions/shareisland.yml @@ -3,6 +3,7 @@ name: Shareisland description: "A general italian tracker" language: it-it + encoding: UTF-8 links: - http://shareisland.org diff --git a/src/Jackett/Definitions/thehorrorcharnel.yml b/src/Jackett/Definitions/thehorrorcharnel.yml index 1e51577ef..003f2924a 100644 --- a/src/Jackett/Definitions/thehorrorcharnel.yml +++ b/src/Jackett/Definitions/thehorrorcharnel.yml @@ -3,6 +3,7 @@ name: The Horror Charnel description: "A Horror, Cult, Sleaze, Sci-Fi & more tracker" language: en-us + encoding: UTF-8 links: - https://horrorcharnel.org diff --git a/src/Jackett/Definitions/torrentsectorcrew.yml b/src/Jackett/Definitions/torrentsectorcrew.yml index 537190d7c..30bef70ce 100644 --- a/src/Jackett/Definitions/torrentsectorcrew.yml +++ b/src/Jackett/Definitions/torrentsectorcrew.yml @@ -3,6 +3,7 @@ name: Torrent Sector Crew description: "A German general tracker" language: de-de + encoding: windows-1252 links: - https://tsctracker.net/ diff --git a/src/Jackett/Definitions/tspate.yml b/src/Jackett/Definitions/tspate.yml index edff1b44b..b9739b479 100644 --- a/src/Jackett/Definitions/tspate.yml +++ b/src/Jackett/Definitions/tspate.yml @@ -3,6 +3,7 @@ name: Tspate description: "A French gerneral tracker" language: fr-fr + encoding: UTF-8 links: - https://www.tspate.me/ diff --git a/src/Jackett/Definitions/uhdbits.yml b/src/Jackett/Definitions/uhdbits.yml index 1604f4f73..2ac1e3b52 100644 --- a/src/Jackett/Definitions/uhdbits.yml +++ b/src/Jackett/Definitions/uhdbits.yml @@ -3,6 +3,7 @@ name: UHDBits description: "A vietnamese general tracker" language: en-us + encoding: UTF-8 links: - https://uhdbits.org/ diff --git a/src/Jackett/Definitions/worldofp2p.yml b/src/Jackett/Definitions/worldofp2p.yml index 6ee4d7a6d..eef1a37f9 100644 --- a/src/Jackett/Definitions/worldofp2p.yml +++ b/src/Jackett/Definitions/worldofp2p.yml @@ -3,6 +3,7 @@ name: WorldOfP2P description: "A general tracker" language: en-us + encoding: UTF-8 links: - https://worldofp2p.net diff --git a/src/Jackett/Indexers/Abstract/AvistazTracker.cs b/src/Jackett/Indexers/Abstract/AvistazTracker.cs index d1bd93c78..ca9c95e41 100644 --- a/src/Jackett/Indexers/Abstract/AvistazTracker.cs +++ b/src/Jackett/Indexers/Abstract/AvistazTracker.cs @@ -40,6 +40,8 @@ namespace Jackett.Indexers p: protectionService, configData: new ConfigurationDataBasicLogin()) { + Encoding = Encoding.GetEncoding("UTF-8"); + AddCategoryMapping(1, TorznabCatType.Movies); AddCategoryMapping(1, TorznabCatType.MoviesForeign); AddCategoryMapping(1, TorznabCatType.MoviesHD); diff --git a/src/Jackett/Indexers/AlphaRatio.cs b/src/Jackett/Indexers/AlphaRatio.cs index 94edcfe20..68afd79e7 100644 --- a/src/Jackett/Indexers/AlphaRatio.cs +++ b/src/Jackett/Indexers/AlphaRatio.cs @@ -38,6 +38,8 @@ namespace Jackett.Indexers downloadBase: "https://alpharatio.cc/torrents.php?action=download&id=", configData: new ConfigurationDataBasicLogin()) { + Encoding = Encoding.GetEncoding("UTF-8"); + AddCategoryMapping(1, TorznabCatType.TVSD); AddCategoryMapping(2, TorznabCatType.TVHD); AddCategoryMapping(6, TorznabCatType.MoviesSD); diff --git a/src/Jackett/Indexers/Andraste.cs b/src/Jackett/Indexers/Andraste.cs index 86a145e1f..fbac1c3f1 100644 --- a/src/Jackett/Indexers/Andraste.cs +++ b/src/Jackett/Indexers/Andraste.cs @@ -38,6 +38,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLoginWithRSSAndDisplay()) { + Encoding = Encoding.GetEncoding("iso-8859-1"); + AddCategoryMapping(9, TorznabCatType.Other); // Anderes AddCategoryMapping(23, TorznabCatType.TVAnime); // Animation - Film &; Serie AddCategoryMapping(1, TorznabCatType.PC); // Appz @@ -121,8 +123,8 @@ namespace Jackett.Indexers } searchUrl += "?" + queryCollection.GetQueryString(); - var response = await RequestBytesWithCookies(searchUrl); - var results = Encoding.GetEncoding("iso-8859-1").GetString(response.Content); + var response = await RequestStringWithCookies(searchUrl); + var results = response.Content; try { CQ dom = results; diff --git a/src/Jackett/Indexers/AnimeBytes.cs b/src/Jackett/Indexers/AnimeBytes.cs index e1cfa3b08..b30e626d5 100644 --- a/src/Jackett/Indexers/AnimeBytes.cs +++ b/src/Jackett/Indexers/AnimeBytes.cs @@ -61,7 +61,7 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataAnimeBytes()) { - + Encoding = Encoding.GetEncoding("UTF-8"); } diff --git a/src/Jackett/Indexers/AnimeTorrents.cs b/src/Jackett/Indexers/AnimeTorrents.cs index 697fe1060..6c9faf913 100644 --- a/src/Jackett/Indexers/AnimeTorrents.cs +++ b/src/Jackett/Indexers/AnimeTorrents.cs @@ -43,6 +43,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLogin()) { + Encoding = Encoding.GetEncoding("UTF-8"); + AddCategoryMapping(1, TorznabCatType.MoviesSD); // Anime Movie AddCategoryMapping(6, TorznabCatType.MoviesHD); // Anime Movie HD AddCategoryMapping(2, TorznabCatType.TVAnime); // Anime Series diff --git a/src/Jackett/Indexers/BB.cs b/src/Jackett/Indexers/BB.cs index c84cc8f8f..c20f6c739 100644 --- a/src/Jackett/Indexers/BB.cs +++ b/src/Jackett/Indexers/BB.cs @@ -44,6 +44,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLogin()) { + Encoding = Encoding.GetEncoding("UTF-8"); + AddCategoryMapping(1, TorznabCatType.Audio); AddCategoryMapping(2, TorznabCatType.PC); AddCategoryMapping(3, TorznabCatType.BooksEbook); diff --git a/src/Jackett/Indexers/BJShare.cs b/src/Jackett/Indexers/BJShare.cs index 5edeea5ac..f9f72f331 100644 --- a/src/Jackett/Indexers/BJShare.cs +++ b/src/Jackett/Indexers/BJShare.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using Newtonsoft.Json.Linq; using System.Collections.Generic; using System; +using System.Text; using System.Globalization; using Jackett.Models.IndexerConfig; using System.Collections.Specialized; @@ -38,7 +39,9 @@ namespace Jackett.Indexers logger: l, p: ps, configData: new ConfigurationDataBasicLoginWithRSSAndDisplay()) - { + { + Encoding = Encoding.GetEncoding("UTF-8"); + AddCategoryMapping(14, TorznabCatType.TVAnime); // Anime AddCategoryMapping(3, TorznabCatType.PC0day); // Aplicativos AddCategoryMapping(8, TorznabCatType.Other); // Apostilas/Tutoriais diff --git a/src/Jackett/Indexers/BaseIndexer.cs b/src/Jackett/Indexers/BaseIndexer.cs index 9c967d6b4..b19aeed57 100644 --- a/src/Jackett/Indexers/BaseIndexer.cs +++ b/src/Jackett/Indexers/BaseIndexer.cs @@ -21,6 +21,7 @@ namespace Jackett.Indexers public string DisplayDescription { get; protected set; } public string DisplayName { get; protected set; } public string Language { get; protected set; } + public Encoding Encoding { get; protected set; } public string ID { get { return GetIndexerID(GetType()); } } public bool IsConfigured { get; protected set; } @@ -239,7 +240,8 @@ namespace Jackett.Indexers { Url = overrideRedirectUrl ?? incomingResponse.RedirectingTo, Referer = referrer, - Cookies = redirRequestCookies + Cookies = redirRequestCookies, + Encoding = Encoding }); Mapper.Map(redirectedResponse, incomingResponse); } @@ -338,7 +340,8 @@ namespace Jackett.Indexers Type = RequestType.GET, Cookies = CookieHeader, Referer = referer, - Headers = headers + Headers = headers, + Encoding = Encoding }; if (cookieOverride != null) @@ -374,7 +377,8 @@ namespace Jackett.Indexers { Url = url, Type = RequestType.GET, - Cookies = cookieOverride ?? CookieHeader + Cookies = cookieOverride ?? CookieHeader, + Encoding = Encoding }; if (cookieOverride != null) @@ -392,7 +396,8 @@ namespace Jackett.Indexers PostData = data, Referer = referer, Headers = headers, - RawBody = rawbody + RawBody = rawbody, + Encoding = Encoding }; if (emulateBrowser.HasValue) @@ -430,7 +435,8 @@ namespace Jackett.Indexers Type = RequestType.POST, Cookies = cookies, Referer = referer, - PostData = data + PostData = data, + Encoding = Encoding }; var response = await webclient.GetString(request); if (accumulateCookies) diff --git a/src/Jackett/Indexers/BestFriends.cs b/src/Jackett/Indexers/BestFriends.cs index b1e19b63d..4d1f9c594 100644 --- a/src/Jackett/Indexers/BestFriends.cs +++ b/src/Jackett/Indexers/BestFriends.cs @@ -11,6 +11,7 @@ using System; using System.Globalization; using Jackett.Models.IndexerConfig; using System.Collections.Specialized; +using System.Text; namespace Jackett.Indexers { @@ -37,6 +38,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataCaptchaLogin()) { + Encoding = Encoding.GetEncoding("iso-8859-1"); + AddCategoryMapping(18, TorznabCatType.TVAnime); // Anime AddCategoryMapping(8, TorznabCatType.PCMac); // Appz MAC AddCategoryMapping(9, TorznabCatType.PC); // Appz other diff --git a/src/Jackett/Indexers/BeyondHD.cs b/src/Jackett/Indexers/BeyondHD.cs index 41bd58eac..aaa7c5a5a 100644 --- a/src/Jackett/Indexers/BeyondHD.cs +++ b/src/Jackett/Indexers/BeyondHD.cs @@ -12,6 +12,7 @@ using System.Threading.Tasks; using Jackett.Models.IndexerConfig; using System.Collections.Specialized; using System.Text.RegularExpressions; +using System.Text; namespace Jackett.Indexers { @@ -36,6 +37,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataLoginLink()) { + Encoding = Encoding.GetEncoding("UTF-8"); + configData.DisplayText.Value = "Go to the general tab of your BeyondHD user profile and create/copy the Login Link."; AddCategoryMapping(37, TorznabCatType.MoviesBluRay); // Movie / Blu-ray diff --git a/src/Jackett/Indexers/BitCityReloaded.cs b/src/Jackett/Indexers/BitCityReloaded.cs index 7dd7f0bed..d32ce58eb 100644 --- a/src/Jackett/Indexers/BitCityReloaded.cs +++ b/src/Jackett/Indexers/BitCityReloaded.cs @@ -9,6 +9,7 @@ using System.Collections.Generic; using CsQuery; using System.Web; using System; +using System.Text; using System.Globalization; using Jackett.Models.IndexerConfig; using System.Collections.Specialized; @@ -38,6 +39,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLoginWithRSSAndDisplay()) { + Encoding = Encoding.GetEncoding("iso-8859-1"); + this.configData.DisplayText.Value = "Only the results from the first search result page are shown, adjust your profile settings to show a reasonable amount (it looks like there's no maximum)."; this.configData.DisplayText.Name = "Notice"; diff --git a/src/Jackett/Indexers/BitMeTV.cs b/src/Jackett/Indexers/BitMeTV.cs index a06a054b9..24ad51daf 100644 --- a/src/Jackett/Indexers/BitMeTV.cs +++ b/src/Jackett/Indexers/BitMeTV.cs @@ -44,6 +44,7 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataCaptchaLogin("Ensure that you have the 'Force SSL' option set to 'yes' in your profile on the BitMeTv webpage.")) { + Encoding = Encoding.GetEncoding("iso-8859-1"); } public override async Task GetConfigurationForSetup() diff --git a/src/Jackett/Indexers/CardigannIndexer.cs b/src/Jackett/Indexers/CardigannIndexer.cs index 6ed7c73c1..7383bdf29 100644 --- a/src/Jackett/Indexers/CardigannIndexer.cs +++ b/src/Jackett/Indexers/CardigannIndexer.cs @@ -170,6 +170,7 @@ namespace Jackett.Indexers DisplayName = Definition.Name; DisplayDescription = Definition.Description; SiteLink = Definition.Links[0]; // TODO: implement alternative links + Encoding = Encoding.GetEncoding(Definition.Encoding); if (!SiteLink.EndsWith("/")) SiteLink += "/"; Language = Definition.Language; @@ -483,7 +484,7 @@ namespace Jackett.Indexers return true; } - protected bool CheckIfLoginIsNeeded(WebClientByteResult Result, IHtmlDocument document) + protected bool CheckIfLoginIsNeeded(WebClientStringResult Result, IHtmlDocument document) { if (Result.IsRedirect) { @@ -764,8 +765,8 @@ namespace Jackett.Indexers searchUrl += "&" + queryCollection.GetQueryString(); // send HTTP request - var response = await RequestBytesWithCookies(searchUrl); - var results = Encoding.GetEncoding(Definition.Encoding).GetString(response.Content); + var response = await RequestStringWithCookies(searchUrl); + var results = response.Content; try { var SearchResultParser = new HtmlParser(); @@ -778,8 +779,8 @@ namespace Jackett.Indexers logger.Info(string.Format("CardigannIndexer ({0}): Relogin required", ID)); await DoLogin(); await TestLogin(); - response = await RequestBytesWithCookies(searchUrl); - results = Encoding.GetEncoding(Definition.Encoding).GetString(response.Content); + response = await RequestStringWithCookies(searchUrl); + results = results = response.Content; SearchResultDocument = SearchResultParser.Parse(results); } diff --git a/src/Jackett/Indexers/DanishBits.cs b/src/Jackett/Indexers/DanishBits.cs index ae98870a9..5c1cc9a37 100644 --- a/src/Jackett/Indexers/DanishBits.cs +++ b/src/Jackett/Indexers/DanishBits.cs @@ -8,6 +8,7 @@ using System; using System.Collections.Generic; using System.Globalization; using System.Linq; +using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; using System.Web; @@ -39,6 +40,8 @@ namespace Jackett.Indexers p: ps, configData: new NxtGnConfigurationData()) { + Encoding = Encoding.GetEncoding("UTF-8"); + // Movies Mapping // DanishBits HD AddCategoryMapping(2, TorznabCatType.MoviesHD); diff --git a/src/Jackett/Indexers/Demonoid.cs b/src/Jackett/Indexers/Demonoid.cs index ccb5d8c71..5e1fd32cc 100644 --- a/src/Jackett/Indexers/Demonoid.cs +++ b/src/Jackett/Indexers/Demonoid.cs @@ -38,6 +38,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLogin()) { + Encoding = Encoding.GetEncoding("UTF-8"); + AddCategoryMapping(3, TorznabCatType.TV); AddCategoryMapping(3, TorznabCatType.TVSD); AddCategoryMapping(3, TorznabCatType.TVHD); diff --git a/src/Jackett/Indexers/DigitalHive.cs b/src/Jackett/Indexers/DigitalHive.cs index a5ba3c812..8bf9c9251 100644 --- a/src/Jackett/Indexers/DigitalHive.cs +++ b/src/Jackett/Indexers/DigitalHive.cs @@ -8,6 +8,7 @@ using NLog; using System; using System.Collections.Generic; using System.Linq; +using System.Text; using System.Threading.Tasks; using Jackett.Models.IndexerConfig; using System.Collections.Specialized; @@ -38,6 +39,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataRecaptchaLogin()) { + Encoding = Encoding.GetEncoding("iso-8859-1"); + AddCategoryMapping(0, TorznabCatType.Other); AddCategoryMapping(48, TorznabCatType.Other); // 0Day AddCategoryMapping(56, TorznabCatType.XXXImageset); // 0Day-Imagesets diff --git a/src/Jackett/Indexers/FileList.cs b/src/Jackett/Indexers/FileList.cs index 400eba6e8..89039ec0c 100644 --- a/src/Jackett/Indexers/FileList.cs +++ b/src/Jackett/Indexers/FileList.cs @@ -42,6 +42,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataFileList()) { + Encoding = Encoding.GetEncoding("UTF-8"); + AddCategoryMapping(24, TorznabCatType.TVAnime); AddCategoryMapping(11, TorznabCatType.Audio); AddCategoryMapping(15, TorznabCatType.TV); diff --git a/src/Jackett/Indexers/Freshon.cs b/src/Jackett/Indexers/Freshon.cs index 43350f839..0b19d87de 100644 --- a/src/Jackett/Indexers/Freshon.cs +++ b/src/Jackett/Indexers/Freshon.cs @@ -44,6 +44,7 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLogin()) { + Encoding = Encoding.GetEncoding("UTF-8"); } public async Task ApplyConfiguration(JToken configJson) diff --git a/src/Jackett/Indexers/FunFile.cs b/src/Jackett/Indexers/FunFile.cs index 1536df469..fb5090df0 100644 --- a/src/Jackett/Indexers/FunFile.cs +++ b/src/Jackett/Indexers/FunFile.cs @@ -6,6 +6,7 @@ using Jackett.Utils.Clients; using Newtonsoft.Json.Linq; using NLog; using System; +using System.Text; using System.Collections.Generic; using System.Threading.Tasks; using Jackett.Models.IndexerConfig; @@ -35,6 +36,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLoginWithRSSAndDisplay()) { + Encoding = Encoding.GetEncoding("iso-8859-1"); + AddCategoryMapping(44, TorznabCatType.TVAnime); // Anime AddCategoryMapping(22, TorznabCatType.PC); // Applications AddCategoryMapping(43, TorznabCatType.AudioAudiobook); // Audio Books diff --git a/src/Jackett/Indexers/Fuzer.cs b/src/Jackett/Indexers/Fuzer.cs index 790ef865c..0752b2676 100644 --- a/src/Jackett/Indexers/Fuzer.cs +++ b/src/Jackett/Indexers/Fuzer.cs @@ -42,6 +42,7 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLogin()) { + Encoding = Encoding.GetEncoding("Windows-1255"); TorznabCaps.Categories.Clear(); AddMultiCategoryMapping(TorznabCatType.Movies, 7, 9, 58, 59, 60, 61, 83); @@ -169,11 +170,10 @@ namespace Jackett.Indexers searchUrl += "text=" + strEncoded + "&category=0&search=1"; } - var data = await RequestBytesWithCookiesAndRetry(searchUrl); - var results = Encoding.GetEncoding("Windows-1255").GetString(data.Content); + var data = await RequestStringWithCookiesAndRetry(searchUrl); try { - CQ dom = results; + CQ dom = data.Content; ReleaseInfo release; int rowCount = 0; @@ -257,7 +257,7 @@ namespace Jackett.Indexers } catch (Exception ex) { - OnParseError(results, ex); + OnParseError(data.Content, ex); } return releases; diff --git a/src/Jackett/Indexers/GhostCity.cs b/src/Jackett/Indexers/GhostCity.cs index 4b44691ef..365e0d6c1 100644 --- a/src/Jackett/Indexers/GhostCity.cs +++ b/src/Jackett/Indexers/GhostCity.cs @@ -10,6 +10,7 @@ using System.Collections.Generic; using System.Collections.Specialized; using System.Globalization; using System.Linq; +using System.Text; using System.Threading.Tasks; using Jackett.Models.IndexerConfig; @@ -37,6 +38,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLoginWithRSSAndDisplay()) { + Encoding = Encoding.GetEncoding("UTF-8"); + this.configData.DisplayText.Value = "Only the results from the first search result page are shown, adjust your profile settings to show the maximum."; this.configData.DisplayText.Name = "Notice"; AddMultiCategoryMapping(TorznabCatType.TVAnime, 8, 34, 35, 36); diff --git a/src/Jackett/Indexers/HD4Free.cs b/src/Jackett/Indexers/HD4Free.cs index b9780b9c6..dee002c21 100644 --- a/src/Jackett/Indexers/HD4Free.cs +++ b/src/Jackett/Indexers/HD4Free.cs @@ -8,6 +8,7 @@ using NLog; using System; using System.Collections.Generic; using System.Linq; +using System.Text; using System.Threading.Tasks; using Jackett.Models.IndexerConfig; @@ -38,6 +39,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataRecaptchaLogin()) { + Encoding = Encoding.GetEncoding("UTF-8"); + AddCategoryMapping(42, TorznabCatType.MoviesSD); // LEGi0N 480p AddCategoryMapping(17, TorznabCatType.MoviesHD); // LEGi0N 720p AddCategoryMapping(16, TorznabCatType.MoviesHD); // LEGi0N 1080p diff --git a/src/Jackett/Indexers/HDSpace.cs b/src/Jackett/Indexers/HDSpace.cs index dae862507..94ee0678a 100644 --- a/src/Jackett/Indexers/HDSpace.cs +++ b/src/Jackett/Indexers/HDSpace.cs @@ -40,6 +40,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLogin()) { + Encoding = Encoding.GetEncoding("UTF-8"); + AddCategoryMapping(15, TorznabCatType.MoviesBluRay); // Movie / Blu-ray AddMultiCategoryMapping(TorznabCatType.MoviesHD, 19, // Movie / 1080p diff --git a/src/Jackett/Indexers/HDTorrents.cs b/src/Jackett/Indexers/HDTorrents.cs index 4e44a8099..a07a4d1c2 100644 --- a/src/Jackett/Indexers/HDTorrents.cs +++ b/src/Jackett/Indexers/HDTorrents.cs @@ -41,6 +41,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLogin()) { + Encoding = Encoding.GetEncoding("UTF-8"); + TorznabCaps.Categories.Clear(); AddCategoryMapping("1", TorznabCatType.MoviesHD);// Movie/Blu-Ray diff --git a/src/Jackett/Indexers/Hebits.cs b/src/Jackett/Indexers/Hebits.cs index 2a771fba2..0af6d37b1 100644 --- a/src/Jackett/Indexers/Hebits.cs +++ b/src/Jackett/Indexers/Hebits.cs @@ -40,6 +40,7 @@ namespace Jackett.Indexers downloadBase: "https://hebits.net/", configData: new ConfigurationDataBasicLogin()) { + Encoding = Encoding.GetEncoding("windows-1255"); AddCategoryMapping(19, TorznabCatType.MoviesSD); AddCategoryMapping(25, TorznabCatType.MoviesOther); // Israeli Content @@ -95,11 +96,10 @@ namespace Jackett.Indexers } } - var response = await RequestBytesWithCookies(searchUrl); - var results = Encoding.GetEncoding("windows-1255").GetString(response.Content); + var response = await RequestStringWithCookies(searchUrl); try { - CQ dom = results; + CQ dom = response.Content; CQ qRows = dom[".browse > div > div"]; @@ -162,7 +162,7 @@ namespace Jackett.Indexers catch (Exception ex) { - OnParseError(results, ex); + OnParseError(response.Content, ex); } return releases; diff --git a/src/Jackett/Indexers/Hounddawgs.cs b/src/Jackett/Indexers/Hounddawgs.cs index 8e7cb5747..d9b0ebe18 100644 --- a/src/Jackett/Indexers/Hounddawgs.cs +++ b/src/Jackett/Indexers/Hounddawgs.cs @@ -43,7 +43,9 @@ namespace Jackett.Indexers p: ps, configData: new NxtGnConfigurationData()) { - AddCategoryMapping(92, TorznabCatType.TV); + Encoding = Encoding.GetEncoding("UTF-8"); + + AddCategoryMapping(92, TorznabCatType.TV); AddCategoryMapping(92, TorznabCatType.TVHD); AddCategoryMapping(92, TorznabCatType.TVWEBDL); diff --git a/src/Jackett/Indexers/HouseOfTorrents.cs b/src/Jackett/Indexers/HouseOfTorrents.cs index 2fe27ff18..e22ce0acb 100644 --- a/src/Jackett/Indexers/HouseOfTorrents.cs +++ b/src/Jackett/Indexers/HouseOfTorrents.cs @@ -8,6 +8,7 @@ using NLog; using System; using System.Collections.Generic; using System.Linq; +using System.Text; using System.Threading.Tasks; using Jackett.Models.IndexerConfig; using System.Collections.Specialized; @@ -39,6 +40,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLoginWithRSSAndDisplay()) { + Encoding = Encoding.GetEncoding("UTF-8"); + AddCategoryMapping(42, TorznabCatType.PCMac); // Applications/Mac AddCategoryMapping(34, TorznabCatType.PC); // Applications/PC AddCategoryMapping(66, TorznabCatType.MoviesForeign); // Foreign diff --git a/src/Jackett/Indexers/IPTorrents.cs b/src/Jackett/Indexers/IPTorrents.cs index 8a769668f..1288be291 100644 --- a/src/Jackett/Indexers/IPTorrents.cs +++ b/src/Jackett/Indexers/IPTorrents.cs @@ -44,6 +44,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataRecaptchaLoginWithAlternateLink()) { + Encoding = Encoding.GetEncoding("UTF-8"); + this.configData.Instructions.Value = this.DisplayName + " has multiple URLs. The default (" + this.SiteLink + ") can be changed by entering a new value in the box below."; this.configData.Instructions.Value += "The following are some known URLs for " + this.DisplayName; this.configData.Instructions.Value += "
  • " + String.Join("
  • ", this.KnownURLs.ToArray()) + "
"; diff --git a/src/Jackett/Indexers/ImmortalSeed.cs b/src/Jackett/Indexers/ImmortalSeed.cs index 9775a218a..22899c79f 100644 --- a/src/Jackett/Indexers/ImmortalSeed.cs +++ b/src/Jackett/Indexers/ImmortalSeed.cs @@ -41,6 +41,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLogin()) { + Encoding = Encoding.GetEncoding("UTF-8"); + AddCategoryMapping(32, TorznabCatType.TVAnime); AddCategoryMapping(47, TorznabCatType.TVSD); AddCategoryMapping(8, TorznabCatType.TVHD); diff --git a/src/Jackett/Indexers/MoreThanTV.cs b/src/Jackett/Indexers/MoreThanTV.cs index 3f6f9a32e..f10735f92 100644 --- a/src/Jackett/Indexers/MoreThanTV.cs +++ b/src/Jackett/Indexers/MoreThanTV.cs @@ -7,6 +7,7 @@ using System; using System.Collections.Generic; using System.Globalization; using System.Linq; +using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; using System.Web; @@ -39,6 +40,7 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLogin()) { + Encoding = Encoding.GetEncoding("UTF-8"); } public async Task ApplyConfiguration(JToken configJson) diff --git a/src/Jackett/Indexers/MyAnonamouse.cs b/src/Jackett/Indexers/MyAnonamouse.cs index 624c8f012..b3e7622e0 100644 --- a/src/Jackett/Indexers/MyAnonamouse.cs +++ b/src/Jackett/Indexers/MyAnonamouse.cs @@ -48,6 +48,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLogin()) { + Encoding = Encoding.GetEncoding("UTF-8"); + AddCategoryMapping("61", TorznabCatType.BooksComics); AddCategoryMapping("91", TorznabCatType.BooksTechnical); AddCategoryMapping("80", TorznabCatType.BooksTechnical); diff --git a/src/Jackett/Indexers/NewRealWorld.cs b/src/Jackett/Indexers/NewRealWorld.cs index e1fb72a89..f2de1fbfa 100644 --- a/src/Jackett/Indexers/NewRealWorld.cs +++ b/src/Jackett/Indexers/NewRealWorld.cs @@ -37,6 +37,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLoginWithRSSAndDisplay()) { + Encoding = Encoding.GetEncoding("iso-8859-1"); + AddCategoryMapping(39, TorznabCatType.TVAnime); // Anime: HD|1080p AddCategoryMapping(38, TorznabCatType.TVAnime); // Anime: HD|720p AddCategoryMapping(1, TorznabCatType.TVAnime); // Anime: SD @@ -146,8 +148,8 @@ namespace Jackett.Indexers searchUrl += "?" + queryCollection.GetQueryString(); - var response = await RequestBytesWithCookies(searchUrl); - var results = Encoding.GetEncoding("iso-8859-1").GetString(response.Content); + var response = await RequestStringWithCookies(searchUrl); + var results = response.Content; try { CQ dom = results; @@ -187,7 +189,6 @@ namespace Jackett.Indexers release.Guid = release.Link; var sizeStr = qSize.Text(); - logger.Error(sizeStr); release.Size = ReleaseInfo.GetBytes(sizeStr.Replace(".", "").Replace(",", ".")); release.Seeders = ParseUtil.CoerceInt(qSeeders.Text()); diff --git a/src/Jackett/Indexers/PirateTheNet.cs b/src/Jackett/Indexers/PirateTheNet.cs index a42800e84..f150cf519 100644 --- a/src/Jackett/Indexers/PirateTheNet.cs +++ b/src/Jackett/Indexers/PirateTheNet.cs @@ -8,6 +8,7 @@ using NLog; using System; using System.Collections.Generic; using System.Linq; +using System.Text; using System.Threading.Tasks; using Jackett.Models.IndexerConfig; using System.Collections.Specialized; @@ -40,6 +41,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLoginWithRSSAndDisplay()) { + Encoding = Encoding.GetEncoding("UTF-8"); + this.configData.DisplayText.Value = "Only the results from the first search result page are shown, adjust your profile settings to show the maximum."; this.configData.DisplayText.Name = "Notice"; diff --git a/src/Jackett/Indexers/Pretome.cs b/src/Jackett/Indexers/Pretome.cs index a91502736..cd5b89668 100644 --- a/src/Jackett/Indexers/Pretome.cs +++ b/src/Jackett/Indexers/Pretome.cs @@ -41,6 +41,7 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataPinNumber()) { + Encoding = Encoding.GetEncoding("iso-8859-1"); AddCategoryMapping("cat[]=22&tags=Windows", TorznabCatType.PC0day); AddCategoryMapping("cat[]=22&tags=MAC", TorznabCatType.PCMac); diff --git a/src/Jackett/Indexers/SceneAccess.cs b/src/Jackett/Indexers/SceneAccess.cs index b202d160b..e84082964 100644 --- a/src/Jackett/Indexers/SceneAccess.cs +++ b/src/Jackett/Indexers/SceneAccess.cs @@ -39,6 +39,7 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLogin()) { + Encoding = Encoding.GetEncoding("UTF-8"); AddCategoryMapping(8, TorznabCatType.MoviesSD); AddCategoryMapping(22, TorznabCatType.MoviesHD); diff --git a/src/Jackett/Indexers/SceneTime.cs b/src/Jackett/Indexers/SceneTime.cs index 53f982767..42d32a83e 100644 --- a/src/Jackett/Indexers/SceneTime.cs +++ b/src/Jackett/Indexers/SceneTime.cs @@ -8,6 +8,7 @@ using NLog; using System; using System.Collections.Generic; using System.Linq; +using System.Text; using System.Threading.Tasks; using Jackett.Models.IndexerConfig; using System.Text.RegularExpressions; @@ -38,6 +39,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataRecaptchaLogin("For best results, change the 'Torrents per page' setting to the maximum in your profile on the SceneTime webpage.")) { + Encoding = Encoding.GetEncoding("iso-8859-1"); + AddCategoryMapping(1, TorznabCatType.MoviesSD); AddCategoryMapping(3, TorznabCatType.MoviesDVD); AddCategoryMapping(47, TorznabCatType.MoviesSD); diff --git a/src/Jackett/Indexers/TVChaosUK.cs b/src/Jackett/Indexers/TVChaosUK.cs index 202fbeb49..65bc6b9f5 100644 --- a/src/Jackett/Indexers/TVChaosUK.cs +++ b/src/Jackett/Indexers/TVChaosUK.cs @@ -47,6 +47,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLoginWithRSS()) { + Encoding = Encoding.GetEncoding("UTF-8"); + AddCategoryMapping(72, TorznabCatType.PC); AddCategoryMapping(86, TorznabCatType.Audio); AddCategoryMapping(87, TorznabCatType.AudioAudiobook); diff --git a/src/Jackett/Indexers/TVVault.cs b/src/Jackett/Indexers/TVVault.cs index 3a7c5be6d..3c2b0ad1b 100644 --- a/src/Jackett/Indexers/TVVault.cs +++ b/src/Jackett/Indexers/TVVault.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using Newtonsoft.Json.Linq; using System.Collections.Generic; using System; +using System.Text; using Jackett.Models.IndexerConfig; using System.Collections.Specialized; using AngleSharp.Parser.Html; @@ -36,6 +37,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLoginWithRSSAndDisplay()) { + Encoding = Encoding.GetEncoding("UTF-8"); + AddCategoryMapping(1, TorznabCatType.TV); AddCategoryMapping(2, TorznabCatType.Movies); } diff --git a/src/Jackett/Indexers/TehConnection.cs b/src/Jackett/Indexers/TehConnection.cs index 976b16aeb..9b80793b1 100644 --- a/src/Jackett/Indexers/TehConnection.cs +++ b/src/Jackett/Indexers/TehConnection.cs @@ -9,6 +9,7 @@ using System; using System.Collections.Generic; using System.Globalization; using System.Linq; +using System.Text; using System.Threading.Tasks; using System.Web; using Jackett.Models.IndexerConfig; @@ -40,6 +41,8 @@ namespace Jackett.Indexers Separate options with a space if using more than one option.
Filter options available:
QualityEncodeOnly
FreeLeechOnly")) { + Encoding = Encoding.GetEncoding("UTF-8"); + AddCategoryMapping(1, TorznabCatType.Movies); AddCategoryMapping(1, TorznabCatType.MoviesForeign); AddCategoryMapping(1, TorznabCatType.MoviesOther); diff --git a/src/Jackett/Indexers/TorrentBytes.cs b/src/Jackett/Indexers/TorrentBytes.cs index cb0a6aca6..3b72f2d90 100644 --- a/src/Jackett/Indexers/TorrentBytes.cs +++ b/src/Jackett/Indexers/TorrentBytes.cs @@ -41,6 +41,7 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLogin()) { + Encoding = Encoding.GetEncoding("iso-8859-1"); AddCategoryMapping(41, TorznabCatType.TV); AddCategoryMapping(33, TorznabCatType.TVSD); diff --git a/src/Jackett/Indexers/TorrentDay.cs b/src/Jackett/Indexers/TorrentDay.cs index 4019553d1..f5cc689fd 100644 --- a/src/Jackett/Indexers/TorrentDay.cs +++ b/src/Jackett/Indexers/TorrentDay.cs @@ -56,6 +56,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataRecaptchaLoginWithAlternateLink()) { + Encoding = Encoding.GetEncoding("UTF-8"); + this.configData.Instructions.Value = this.DisplayName + " has multiple URLs. The default (" + this.SiteLink + ") can be changed by entering a new value in the box below."; this.configData.Instructions.Value += "The following are some known URLs for " + this.DisplayName; this.configData.Instructions.Value += "
  • " + String.Join("
  • ", this.KnownURLs.ToArray()) + "
"; diff --git a/src/Jackett/Indexers/TorrentHeaven.cs b/src/Jackett/Indexers/TorrentHeaven.cs index 0ea8810b4..393a91749 100644 --- a/src/Jackett/Indexers/TorrentHeaven.cs +++ b/src/Jackett/Indexers/TorrentHeaven.cs @@ -40,6 +40,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataCaptchaLogin()) { + Encoding = Encoding.GetEncoding("iso-8859-1"); + AddCategoryMapping(1, TorznabCatType.PCGames); // GAMES/PC AddCategoryMapping(3, TorznabCatType.Console); // GAMES/Sonstige AddCategoryMapping(59, TorznabCatType.ConsolePS4); // GAMES/PlayStation @@ -178,8 +180,8 @@ namespace Jackett.Indexers } searchUrl += "?" + queryCollection.GetQueryString(); - var response = await RequestBytesWithCookies(searchUrl); - var results = Encoding.GetEncoding("iso-8859-1").GetString(response.Content); + var response = await RequestStringWithCookies(searchUrl); + var results = response.Content; var TitleRegexp = new Regex(@"^return buildTable\('(.*?)',\s+"); try { diff --git a/src/Jackett/Indexers/TorrentLeech.cs b/src/Jackett/Indexers/TorrentLeech.cs index cedf8cd87..fbd64f8ed 100644 --- a/src/Jackett/Indexers/TorrentLeech.cs +++ b/src/Jackett/Indexers/TorrentLeech.cs @@ -41,6 +41,7 @@ namespace Jackett.Indexers downloadBase: "https://www.torrentleech.org/download/", configData: new ConfigurationDataBasicLogin("For best results, change the 'Default Number of Torrents per Page' setting to the maximum in your profile on the TorrentLeech webpage.")) { + Encoding = Encoding.GetEncoding("iso-8859-1"); AddCategoryMapping(8, TorznabCatType.MoviesSD); // cam AddCategoryMapping(9, TorznabCatType.MoviesSD); //ts diff --git a/src/Jackett/Indexers/TorrentNetwork.cs b/src/Jackett/Indexers/TorrentNetwork.cs index 5d7807be3..fd5710e29 100644 --- a/src/Jackett/Indexers/TorrentNetwork.cs +++ b/src/Jackett/Indexers/TorrentNetwork.cs @@ -37,6 +37,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLoginWithRSSAndDisplay()) { + Encoding = Encoding.GetEncoding("UTF-8"); + AddCategoryMapping(1, TorznabCatType.AudioAudiobook); // aBook AddCategoryMapping(4, TorznabCatType.PCMac); // App|Mac AddCategoryMapping(5, TorznabCatType.PC); // App|Win @@ -130,8 +132,8 @@ namespace Jackett.Indexers } searchUrl += "?" + queryCollection.GetQueryString(); - var response = await RequestBytesWithCookies(searchUrl); - var results = Encoding.GetEncoding("iso-8859-1").GetString(response.Content); + var response = await RequestStringWithCookies(searchUrl); + var results = response.Content; try { CQ dom = results; diff --git a/src/Jackett/Indexers/TorrentShack.cs b/src/Jackett/Indexers/TorrentShack.cs index 90b17dffa..bef620c93 100644 --- a/src/Jackett/Indexers/TorrentShack.cs +++ b/src/Jackett/Indexers/TorrentShack.cs @@ -41,6 +41,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLogin()) { + Encoding = Encoding.GetEncoding("UTF-8"); + AddCategoryMapping(600, TorznabCatType.TVHD); // TV/HD AddCategoryMapping(960, TorznabCatType.MoviesForeign); // Foreign AddCategoryMapping(300, TorznabCatType.MoviesHD); // Movies/HD diff --git a/src/Jackett/Indexers/TorrentSyndikat.cs b/src/Jackett/Indexers/TorrentSyndikat.cs index e6cfe6e85..e21c4c77b 100644 --- a/src/Jackett/Indexers/TorrentSyndikat.cs +++ b/src/Jackett/Indexers/TorrentSyndikat.cs @@ -8,6 +8,7 @@ using NLog; using System; using System.Collections.Generic; using System.Linq; +using System.Text; using System.Threading.Tasks; using Jackett.Models.IndexerConfig; using System.Collections.Specialized; @@ -40,6 +41,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLoginWithRSSAndDisplay()) { + Encoding = Encoding.GetEncoding("UTF-8"); + this.configData.DisplayText.Value = "Only the results from the first search result page are shown, adjust your profile settings to show the maximum."; this.configData.DisplayText.Name = "Notice"; diff --git a/src/Jackett/Indexers/TransmitheNet.cs b/src/Jackett/Indexers/TransmitheNet.cs index 441710724..4cc16152f 100644 --- a/src/Jackett/Indexers/TransmitheNet.cs +++ b/src/Jackett/Indexers/TransmitheNet.cs @@ -9,6 +9,7 @@ using System.Collections.Generic; using System.Globalization; using System.Threading.Tasks; using System.Web; +using System.Text; using Jackett.Models.IndexerConfig; using AngleSharp.Parser.Html; using System.Text.RegularExpressions; @@ -37,6 +38,7 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLogin("For best results, change the 'Torrents per page' setting to 100 in your profile on the TTN webpage.")) { + Encoding = Encoding.GetEncoding("UTF-8"); } public async Task ApplyConfiguration(JToken configJson) diff --git a/src/Jackett/Indexers/XSpeeds.cs b/src/Jackett/Indexers/XSpeeds.cs index 07ef0f90c..9340904bd 100644 --- a/src/Jackett/Indexers/XSpeeds.cs +++ b/src/Jackett/Indexers/XSpeeds.cs @@ -9,6 +9,7 @@ using System; using System.Collections.Generic; using System.Globalization; using System.Linq; +using System.Text; using System.Threading.Tasks; using Jackett.Models.IndexerConfig; using System.Text.RegularExpressions; @@ -43,6 +44,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLoginWithRSSAndDisplay()) { + Encoding = Encoding.GetEncoding("UTF-8"); + configData.DisplayText.Value = "Expect an initial delay (often around 10 seconds) due to XSpeeds CloudFlare DDoS protection"; configData.DisplayText.Name = "Notice"; diff --git a/src/Jackett/Indexers/myAmity.cs b/src/Jackett/Indexers/myAmity.cs index e89f85b89..b463b5ebc 100644 --- a/src/Jackett/Indexers/myAmity.cs +++ b/src/Jackett/Indexers/myAmity.cs @@ -37,6 +37,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataBasicLoginWithRSSAndDisplay()) { + Encoding = Encoding.GetEncoding("UTF-8"); + AddCategoryMapping(20, TorznabCatType.PC); // Apps - PC AddCategoryMapping(24, TorznabCatType.AudioAudiobook); // Audio - Hoerbuch/-spiel AddCategoryMapping(22, TorznabCatType.Audio); // Audio - Musik @@ -108,8 +110,8 @@ namespace Jackett.Indexers } searchUrl += "?" + queryCollection.GetQueryString(); - var response = await RequestBytesWithCookies(searchUrl); - var results = Encoding.UTF8.GetString(response.Content); + var response = await RequestStringWithCookies(searchUrl); + var results = response.Content; try { CQ dom = results; diff --git a/src/Jackett/Indexers/x264.cs b/src/Jackett/Indexers/x264.cs index 43fbbb021..7507c6575 100644 --- a/src/Jackett/Indexers/x264.cs +++ b/src/Jackett/Indexers/x264.cs @@ -8,6 +8,7 @@ using NLog; using System; using System.Collections.Generic; using System.Linq; +using System.Text; using System.Threading.Tasks; using Jackett.Models.IndexerConfig; using System.Collections.Specialized; @@ -37,6 +38,8 @@ namespace Jackett.Indexers p: ps, configData: new ConfigurationDataRecaptchaLogin()) { + Encoding = Encoding.GetEncoding("iso-8859-1"); + AddCategoryMapping(20, TorznabCatType.Movies); // Movies&TV/Sources AddCategoryMapping(53, TorznabCatType.MoviesHD); // Movies/1080p AddCategoryMapping(30, TorznabCatType.MoviesHD); // Movies/576p diff --git a/src/Jackett/Utils/Clients/HttpWebClient.cs b/src/Jackett/Utils/Clients/HttpWebClient.cs index 86aee6c0c..6565f05d1 100644 --- a/src/Jackett/Utils/Clients/HttpWebClient.cs +++ b/src/Jackett/Utils/Clients/HttpWebClient.cs @@ -9,23 +9,21 @@ using System.Linq; using System.Net; using System.Net.Http; using System.Text; +using System.Text.RegularExpressions; using System.Threading.Tasks; namespace Jackett.Utils.Clients { public class HttpWebClient : IWebClient { - Logger logger; - IConfigurationService configService; - - public HttpWebClient(Logger l, IConfigurationService c) - { - logger = l; - configService = c; + public HttpWebClient(IProcessService p, Logger l, IConfigurationService c) + : base(p: p, + l: l, + c: c) + { } - - public void Init() + override public void Init() { if (Startup.IgnoreSslErrors == true) { @@ -34,30 +32,7 @@ namespace Jackett.Utils.Clients } } - public async Task GetBytes(WebRequest request) - { - logger.Debug(string.Format("WindowsWebClient:GetBytes(Url:{0})", request.Url)); - var result = await Run(request); - logger.Debug(string.Format("WindowsWebClient: Returning {0} => {1} bytes", result.Status, (result.Content == null ? "" : result.Content.Length.ToString()))); - return result; - } - - public async Task GetString(WebRequest request) - { - logger.Debug(string.Format("WindowsWebClient:GetString(Url:{0})", request.Url)); - var result = await Run(request); - logger.Debug(string.Format("WindowsWebClient: Returning {0} => {1}", result.Status, (result.Content == null ? "" : Encoding.UTF8.GetString(result.Content)))); - WebClientStringResult stringResult = Mapper.Map(result); - string[] server; - if (stringResult.Headers.TryGetValue("server", out server)) - { - if (server[0] == "cloudflare-nginx") - stringResult.Content = BrowserUtil.DecodeCloudFlareProtectedEmailFromHTML(stringResult.Content); - } - return stringResult; - } - - private async Task Run(WebRequest webRequest) + override protected async Task Run(WebRequest webRequest) { ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072; diff --git a/src/Jackett/Utils/Clients/IWebClient.cs b/src/Jackett/Utils/Clients/IWebClient.cs index 24ad8dfcb..ad1fb6722 100644 --- a/src/Jackett/Utils/Clients/IWebClient.cs +++ b/src/Jackett/Utils/Clients/IWebClient.cs @@ -1,16 +1,93 @@ -using Jackett.Models; +using AutoMapper; +using Jackett.Models; +using Jackett.Services; +using NLog; using System; using System.Collections.Generic; using System.Linq; using System.Text; +using System.Text.RegularExpressions; using System.Threading.Tasks; namespace Jackett.Utils.Clients { - public interface IWebClient + public abstract class IWebClient { - Task GetString(WebRequest request); - Task GetBytes(WebRequest request); - void Init(); + protected Logger logger; + protected IConfigurationService configService; + protected IProcessService processService; + + public IWebClient(IProcessService p, Logger l, IConfigurationService c) + { + processService = p; + logger = l; + configService = c; + } + + virtual public async Task GetBytes(WebRequest request) + { + logger.Debug(string.Format("IWebClient.GetBytes(Url:{0})", request.Url)); + var result = await Run(request); + logger.Debug(string.Format("IWebClient: Returning {0} => {1} bytes", result.Status, (result.Content == null ? "" : result.Content.Length.ToString()))); + return result; + } + + virtual public async Task GetString(WebRequest request) + { + logger.Debug(string.Format("IWebClient.GetString(Url:{0})", request.Url)); + var result = await Run(request); + WebClientStringResult stringResult = Mapper.Map(result); + Encoding encoding = null; + if (request.Encoding != null) + { + encoding = request.Encoding; + } + else if (result.Headers.ContainsKey("content-type")) + { + Regex CharsetRegex = new Regex(@"charset=([\w-]+)", RegexOptions.Compiled); + var CharsetRegexMatch = CharsetRegex.Match(result.Headers["content-type"][0]); + if (CharsetRegexMatch.Success) + { + var charset = CharsetRegexMatch.Groups[1].Value; + try + { + encoding = Encoding.GetEncoding(charset); + } + catch (Exception ex) + { + logger.Error(string.Format("IWebClient.GetString(Url:{0}): Error loading encoding {0} based on header {1}: {2}", request.Url, charset, result.Headers["content-type"][0], ex)); + } + } + else + { + logger.Error(string.Format("IWebClient.GetString(Url:{0}): Got header without charset: {0}", request.Url, result.Headers["content-type"][0])); + } + } + + if (encoding == null) + { + logger.Error(string.Format("IWebClient.GetString(Url:{0}): No encoding detected, defaulting to UTF-8", request.Url)); + encoding = Encoding.UTF8; + } + + string decodedContent = null; + if (result.Content != null) + decodedContent = encoding.GetString(result.Content); + + stringResult.Content = decodedContent; + logger.Debug(string.Format("IWebClient: Returning {0} => {1}", result.Status, (decodedContent == null ? "" : decodedContent))); + + string[] server; + if (stringResult.Headers.TryGetValue("server", out server)) + { + if (server[0] == "cloudflare-nginx") + stringResult.Content = BrowserUtil.DecodeCloudFlareProtectedEmailFromHTML(stringResult.Content); + } + return stringResult; + } + + virtual protected async Task Run(WebRequest webRequest) { return null; } + + abstract public void Init(); } } diff --git a/src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs b/src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs index 88271e978..4d5eff60c 100644 --- a/src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs +++ b/src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs @@ -17,34 +17,11 @@ namespace Jackett.Utils.Clients { public class UnixLibCurlWebClient : IWebClient { - private Logger logger; - - public UnixLibCurlWebClient(Logger l) + public UnixLibCurlWebClient(IProcessService p, Logger l, IConfigurationService c) + : base(p: p, + l: l, + c: c) { - logger = l; - } - - public async Task GetBytes(WebRequest request) - { - logger.Debug(string.Format("UnixLibCurlWebClient:GetBytes(Url:{0})", request.Url)); - var result = await RunCloudFlare(request); - logger.Debug(string.Format("UnixLibCurlWebClient:GetBytes Returning {0} => {1} bytes", result.Status, (result.Content == null ? "" : result.Content.Length.ToString()))); - return result; - } - - public async Task GetString(WebRequest request) - { - logger.Debug(string.Format("UnixLibCurlWebClient:GetString(Url:{0})", request.Url)); - var result = await RunCloudFlare(request); - logger.Debug(string.Format("UnixLibCurlWebClient:GetString Returning {0} => {1}", result.Status, (result.Content == null ? "" : Encoding.UTF8.GetString(result.Content)))); - WebClientStringResult stringResult = Mapper.Map(result); - string[] server; - if (stringResult.Headers.TryGetValue("server", out server)) - { - if (server[0] == "cloudflare-nginx") - stringResult.Content = BrowserUtil.DecodeCloudFlareProtectedEmailFromHTML(stringResult.Content); - } - return stringResult; } private string CloudFlareChallengeSolverSolve(string challengePageContent, Uri uri) @@ -54,7 +31,7 @@ namespace Jackett.Utils.Clients return clearanceUri; } - public void Init() + override public void Init() { try { @@ -117,7 +94,7 @@ namespace Jackett.Utils.Clients return result; } - private async Task Run(WebRequest request) + override protected async Task Run(WebRequest request) { Jackett.CurlHelper.CurlResponse response; if (request.Type == RequestType.GET) diff --git a/src/Jackett/Utils/Clients/UnixSafeCurlWebClient.cs b/src/Jackett/Utils/Clients/UnixSafeCurlWebClient.cs index cecef59ea..81192b3c7 100644 --- a/src/Jackett/Utils/Clients/UnixSafeCurlWebClient.cs +++ b/src/Jackett/Utils/Clients/UnixSafeCurlWebClient.cs @@ -16,38 +16,18 @@ namespace Jackett.Utils.Clients { public class UnixSafeCurlWebClient : IWebClient { - IProcessService processService; - Logger logger; - IConfigurationService configService; - public UnixSafeCurlWebClient(IProcessService p, Logger l, IConfigurationService c) - { - processService = p; - logger = l; - configService = c; - } - - public void Init() + : base(p: p, + l: l, + c: c) { } - public async Task GetBytes(WebRequest request) + override public void Init() { - logger.Debug(string.Format("UnixSafeCurlWebClient:GetBytes(Url:{0})", request.Url)); - var result = await Run(request); - logger.Debug(string.Format("UnixSafeCurlWebClient: Returning {0} => {1} bytes", result.Status, (result.Content == null ? "" : result.Content.Length.ToString()))); - return result; } - public async Task GetString(WebRequest request) - { - logger.Debug(string.Format("UnixSafeCurlWebClient:GetString(Url:{0})", request.Url)); - var result = await Run(request); - logger.Debug(string.Format("UnixSafeCurlWebClient: Returning {0} => {1}", result.Status, (result.Content == null ? "" : Encoding.UTF8.GetString(result.Content)))); - return Mapper.Map(result); - } - - private async Task Run(WebRequest request) + override protected async Task Run(WebRequest request) { var args = new StringBuilder(); if (Startup.ProxyConnection != null) diff --git a/src/Jackett/Utils/Clients/WebRequest.cs b/src/Jackett/Utils/Clients/WebRequest.cs index 32b791947..0f1aefd51 100644 --- a/src/Jackett/Utils/Clients/WebRequest.cs +++ b/src/Jackett/Utils/Clients/WebRequest.cs @@ -15,6 +15,7 @@ namespace Jackett.Utils.Clients Type = RequestType.GET; Headers = new Dictionary(); EmulateBrowser = true; + Encoding Encoding; } public WebRequest(string url) @@ -24,6 +25,7 @@ namespace Jackett.Utils.Clients Url = url; Headers = new Dictionary(); EmulateBrowser = true; + Encoding Encoding; } public string Url { get; set; } @@ -33,6 +35,7 @@ namespace Jackett.Utils.Clients public RequestType Type { get; set; } public string RawBody { get; set; } public bool EmulateBrowser { get; set; } + public Encoding Encoding { get; set; } /// /// Warning this is only implemented on HTTPWebClient currently! @@ -76,6 +79,7 @@ namespace Jackett.Utils.Clients other.Referer == Referer && other.Cookies == Cookies && other.Type == Type && + other.Encoding == Encoding && postDataSame; }