diff --git a/README.md b/README.md index 8c38e4ce8..cc8422dd4 100644 --- a/README.md +++ b/README.md @@ -1,97 +1,98 @@ -## Jackett - -#### Download -Downloads on the [Releases page](https://github.com/zone117x/Jackett/releases) - - -#### Overview -This software creates a [Torznab](https://github.com/Sonarr/Sonarr/wiki/Implementing-a-Torznab-indexer) (with [nZEDb](https://github.com/nZEDb/nZEDb/blob/master/docs/newznab_api_specification.txt) category numbering) and [TorrentPotato](https://github.com/RuudBurger/CouchPotatoServer/wiki/Couchpotato-torrent-provider) API server on your machine. Torznab enables software such as [Sonarr](https://sonarr.tv) to access data from your favorite indexers in a similar fashion to rss but with added features such as searching. TorrentPotato is an interface accessible to [CouchPotato](https://couchpota.to/). - -Jackett works as a proxy server: it translates queries from apps (Sonarr, SickRage, CouchPotato, Mylar, etc) into tracker-site-specific http queries, parses the html response, then sends results back to the requesting software. This allows for getting recent uploads (like RSS) and performing searches. Jackett is a single repository of maintained indexer scraping & translation logic - removing the burden from other apps. - -We were previously focused on TV but are working on extending searches to allow for searching other items such as movies, comics, and music. - - -#### Supported Systems -* Windows using .NET 4.5 -* Linux and OSX using Mono 4 (v3 should work but you may experience crashes). - - -#### Supported Trackers - * [AlphaRatio](https://alpharatio.cc/) - * [AnimeBytes](https://animebytes.tv/) - * [Avistaz](https://avistaz.to/) - * [BakaBT](http://bakabt.me/) - * [bB](http://reddit.com/r/baconbits) - * [BeyondHD](https://beyondhd.me/) - * [BIT-HDTV](https://www.bit-hdtv.com) - * [BitMeTV](http://www.bitmetv.org/) - * [Demonoid](http://www.demonoid.pw/) - * [EuTorrents](https://eutorrents.to/) - * [FileList](http://filelist.ro/) - * [FrenchTorrentDb](http://www.frenchtorrentdb.com/) - * [Freshon](https://freshon.tv/) - * [HD-Space](https://hd-space.org/) - * [HD-Torrents.org](https://hd-torrents.org/) - * [Immortalseed.me](http://immortalseed.me) - * [IPTorrents](https://iptorrents.com/) - * [MoreThan.tv](https://morethan.tv/) - * [pretome](https://pretome.info) - * [PrivateHD](https://privatehd.to/) - * [RARGB](https://rarbg.to/) - * [RuTor](http://rutor.org/) - * [SceneAccess](https://sceneaccess.eu/login) - * [SceneTime](https://www.scenetime.com/) - * [ShowRSS](https://showrss.info/) - * [Strike](https://getstrike.net/) - * [T411](http://www.t411.io/) - * [The Pirate Bay](https://thepiratebay.se/) - * [TorrentBytes](https://www.torrentbytes.net/) - * [TorrentDay](https://torrentday.eu/) - * [TorrentLeech](http://www.torrentleech.org/) - * [TorrentShack](http://torrentshack.me/) - * [Torrentz](https://torrentz.eu/) - * [TV Chaos UK](https://tvchaosuk.com/) - -#### Installation on Linux/OSX - 1. Install [Mono 4](http://www.mono-project.com/download/) or better - 2. Install libcurl: - * Debian/Ubunutu: apt-get install libcurl-dev - * Redhat/Fedora: yum install libcurl-devel - * For other distros see the [Curl docs](http://curl.haxx.se/dlwiz/?type=devel). - 3. Download and extract the latest ```.tar.bz2``` release from the [website](http://jackett.net/Download) and run Jackett using mono with the command "mono JackettConsole.exe". - - -#### Installation on Windows - -Grab the latest release from the [web site](http://jackett.net/Download). - -We recommend you install Jackett as a Windows service using the supplied installer. When installed as a service the tray icon acts as a way to open/start/stop Jackett. If you opted to not install it as a service then Jackett will run its web server from the tray tool. - -Jackett can also be run from the command line using JackettConsole.exe if you would like to see log messages (Ensure the server isn't already running from the tray/service). - - -#### Troubleshooting - -* Command line switches - -You can pass various options when running via the command line, see --help for details. - -* Unable to connect to certain trackers on Linux - -Try running with the "--SSLFix true" if you are on Redhat/Fedora/NNS based libcurl. If the tracker is currently configured try removing it and adding it again. Alternatively try running with a different client via --UseClient (Warning: safecurl just executes curl and your details may be seen from the process list). - -* Enable logging - -You can get additional logging with the switches "-t -l". Please post logs if you are unable to resolve your issue with these switches ensuring to remove your username/password/cookies. - - -### Additional Trackers -Jackett's framework allows our team (and any other volunteering dev) to implement new trackers in an hour or two. If you'd like support for a new tracker then feel free to leave a request on the [issues page](https://github.com/zone117x/Jackett/issues) or contact us on IRC (see below). Pull requests must be made to the develop branch. - -### Contact & Support -Use the github issues pages or talk to us directly at: [irc.freenode.net#jackett](http://webchat.freenode.net/?channels=#jackett). - -### Screenshots - +## Jackett + +#### Download +Downloads on the [Releases page](https://github.com/zone117x/Jackett/releases) + + +#### Overview +This software creates a [Torznab](https://github.com/Sonarr/Sonarr/wiki/Implementing-a-Torznab-indexer) (with [nZEDb](https://github.com/nZEDb/nZEDb/blob/master/docs/newznab_api_specification.txt) category numbering) and [TorrentPotato](https://github.com/RuudBurger/CouchPotatoServer/wiki/Couchpotato-torrent-provider) API server on your machine. Torznab enables software such as [Sonarr](https://sonarr.tv) to access data from your favorite indexers in a similar fashion to rss but with added features such as searching. TorrentPotato is an interface accessible to [CouchPotato](https://couchpota.to/). + +Jackett works as a proxy server: it translates queries from apps (Sonarr, SickRage, CouchPotato, Mylar, etc) into tracker-site-specific http queries, parses the html response, then sends results back to the requesting software. This allows for getting recent uploads (like RSS) and performing searches. Jackett is a single repository of maintained indexer scraping & translation logic - removing the burden from other apps. + +We were previously focused on TV but are working on extending searches to allow for searching other items such as movies, comics, and music. + + +#### Supported Systems +* Windows using .NET 4.5 +* Linux and OSX using Mono 4 (v3 should work but you may experience crashes). + + +#### Supported Trackers + * [AlphaRatio](https://alpharatio.cc/) + * [AnimeBytes](https://animebytes.tv/) + * [AnimeTorrents](http://animetorrents.me/) + * [Avistaz](https://avistaz.to/) + * [BakaBT](http://bakabt.me/) + * [bB](http://reddit.com/r/baconbits) + * [BeyondHD](https://beyondhd.me/) + * [BIT-HDTV](https://www.bit-hdtv.com) + * [BitMeTV](http://www.bitmetv.org/) + * [Demonoid](http://www.demonoid.pw/) + * [EuTorrents](https://eutorrents.to/) + * [FileList](http://filelist.ro/) + * [FrenchTorrentDb](http://www.frenchtorrentdb.com/) + * [Freshon](https://freshon.tv/) + * [HD-Space](https://hd-space.org/) + * [HD-Torrents.org](https://hd-torrents.org/) + * [Immortalseed.me](http://immortalseed.me) + * [IPTorrents](https://iptorrents.com/) + * [MoreThan.tv](https://morethan.tv/) + * [pretome](https://pretome.info) + * [PrivateHD](https://privatehd.to/) + * [RARGB](https://rarbg.to/) + * [RuTor](http://rutor.org/) + * [SceneAccess](https://sceneaccess.eu/login) + * [SceneTime](https://www.scenetime.com/) + * [ShowRSS](https://showrss.info/) + * [Strike](https://getstrike.net/) + * [T411](http://www.t411.io/) + * [The Pirate Bay](https://thepiratebay.se/) + * [TorrentBytes](https://www.torrentbytes.net/) + * [TorrentDay](https://torrentday.eu/) + * [TorrentLeech](http://www.torrentleech.org/) + * [TorrentShack](http://torrentshack.me/) + * [Torrentz](https://torrentz.eu/) + * [TV Chaos UK](https://tvchaosuk.com/) + +#### Installation on Linux/OSX + 1. Install [Mono 4](http://www.mono-project.com/download/) or better + 2. Install libcurl: + * Debian/Ubunutu: apt-get install libcurl-dev + * Redhat/Fedora: yum install libcurl-devel + * For other distros see the [Curl docs](http://curl.haxx.se/dlwiz/?type=devel). + 3. Download and extract the latest ```.tar.bz2``` release from the [website](http://jackett.net/Download) and run Jackett using mono with the command "mono JackettConsole.exe". + + +#### Installation on Windows + +Grab the latest release from the [web site](http://jackett.net/Download). + +We recommend you install Jackett as a Windows service using the supplied installer. When installed as a service the tray icon acts as a way to open/start/stop Jackett. If you opted to not install it as a service then Jackett will run its web server from the tray tool. + +Jackett can also be run from the command line using JackettConsole.exe if you would like to see log messages (Ensure the server isn't already running from the tray/service). + + +#### Troubleshooting + +* Command line switches + +You can pass various options when running via the command line, see --help for details. + +* Unable to connect to certain trackers on Linux + +Try running with the "--SSLFix true" if you are on Redhat/Fedora/NNS based libcurl. If the tracker is currently configured try removing it and adding it again. Alternatively try running with a different client via --UseClient (Warning: safecurl just executes curl and your details may be seen from the process list). + +* Enable logging + +You can get additional logging with the switches "-t -l". Please post logs if you are unable to resolve your issue with these switches ensuring to remove your username/password/cookies. + + +### Additional Trackers +Jackett's framework allows our team (and any other volunteering dev) to implement new trackers in an hour or two. If you'd like support for a new tracker then feel free to leave a request on the [issues page](https://github.com/zone117x/Jackett/issues) or contact us on IRC (see below). Pull requests must be made to the develop branch. + +### Contact & Support +Use the github issues pages or talk to us directly at: [irc.freenode.net#jackett](http://webchat.freenode.net/?channels=#jackett). + +### Screenshots + ![screenshot](http://i.imgur.com/t1sVva6.png "screenshot") \ No newline at end of file diff --git a/src/Jackett/Content/logos/animetorrents.png b/src/Jackett/Content/logos/animetorrents.png new file mode 100644 index 000000000..24c836e4b Binary files /dev/null and b/src/Jackett/Content/logos/animetorrents.png differ diff --git a/src/Jackett/Indexers/AnimeTorrents.cs b/src/Jackett/Indexers/AnimeTorrents.cs new file mode 100644 index 000000000..48a1fa95d --- /dev/null +++ b/src/Jackett/Indexers/AnimeTorrents.cs @@ -0,0 +1,176 @@ +using CsQuery; +using Jackett.Models; +using Jackett.Services; +using Jackett.Utils; +using Jackett.Utils.Clients; +using Newtonsoft.Json.Linq; +using NLog; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using System.Web; +using Jackett.Models.IndexerConfig; +using System.Collections.Specialized; +using System.Globalization; + +namespace Jackett.Indexers +{ + public class AnimeTorrents : BaseIndexer, IIndexer + { + private string LoginUrl { get { return SiteLink + "login.php"; } } + private string SearchUrl { get { return SiteLink + "ajax/torrents_data.php"; } } + private string SearchUrlReferer { get { return SiteLink + "torrents.php?cat=0&searchin=filename&search="; } } + + new ConfigurationDataBasicLogin configData + { + get { return (ConfigurationDataBasicLogin)base.configData; } + set { base.configData = value; } + } + + public AnimeTorrents(IIndexerManagerService i, HttpWebClient c, Logger l, IProtectionService ps) + : base(name: "AnimeTorrents", + description: "Definitive source for anime and manga", + link: "http://animetorrents.me/", + caps: new TorznabCapabilities(), + manager: i, + client: c, // Forced HTTP client for custom headers + logger: l, + p: ps, + configData: new ConfigurationDataBasicLogin()) + { + AddCategoryMapping(1, TorznabCatType.MoviesSD); // Anime Movie + AddCategoryMapping(6, TorznabCatType.MoviesHD); // Anime Movie HD + AddCategoryMapping(2, TorznabCatType.TVAnime); // Anime Series + AddCategoryMapping(7, TorznabCatType.TVAnime); // Anime Series HD + AddCategoryMapping(5, TorznabCatType.XXXDVD); // Hentai (censored) + AddCategoryMapping(9, TorznabCatType.XXXDVD); // Hentai (censored) HD + AddCategoryMapping(4, TorznabCatType.XXXDVD); // Hentai (un-censored) + AddCategoryMapping(8, TorznabCatType.XXXDVD); // Hentai (un-censored) HD + AddCategoryMapping(13, TorznabCatType.BooksForeign); // Light Novel + AddCategoryMapping(3, TorznabCatType.BooksComics); // Manga + AddCategoryMapping(10, TorznabCatType.BooksComics); // Manga 18+ + AddCategoryMapping(11, TorznabCatType.TVAnime); // OVA + AddCategoryMapping(12, TorznabCatType.TVAnime); // OVA HD + AddCategoryMapping(14, TorznabCatType.BooksComics); // Doujin Anime + AddCategoryMapping(15, TorznabCatType.XXXDVD); // Doujin Anime 18+ + AddCategoryMapping(16, TorznabCatType.AudioForeign); // Doujin Music + AddCategoryMapping(17, TorznabCatType.BooksComics); // Doujinshi + AddCategoryMapping(18, TorznabCatType.BooksComics); // Doujinshi 18+ + AddCategoryMapping(19, TorznabCatType.Audio); // OST + } + + public async Task ApplyConfiguration(JToken configJson) + { + configData.LoadValuesFromJson(configJson); + var pairs = new Dictionary { + { "username", configData.Username.Value }, + { "password", configData.Password.Value }, + { "form", "login" }, + { "rememberme[]", "1" } + }; + + var loginPage = await RequestStringWithCookiesAndRetry(LoginUrl, null, null); + + var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, SearchUrl, SiteLink); + await ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("logout.php"), () => + { + CQ dom = result.Content; + var errorMessage = dom[".ui-state-error"].Text().Trim(); + throw new ExceptionWithConfigData(errorMessage, configData); + }); + + return IndexerConfigurationStatus.RequiresTesting; + } + + public async Task> PerformQuery(TorznabQuery query) + { + var releases = new List(); + var searchString = query.GetQueryString(); + var searchUrl = SearchUrl; + var queryCollection = new NameValueCollection(); + + queryCollection.Add("total", "146"); // Not sure what this is about but its required! + + var cat = "0"; + var queryCats = MapTorznabCapsToTrackers(query); + if (queryCats.Count == 1) + { + cat = queryCats.First().ToString(); + } + + queryCollection.Add("cat", cat); + queryCollection.Add("searchin", "filename"); + queryCollection.Add("search", searchString); + queryCollection.Add("page", "1"); + searchUrl += "?" + queryCollection.GetQueryString(); + + var extraHeaders = new Dictionary() + { + { "X-Requested-With", "XMLHttpRequest" } + }; + + var response = await RequestStringWithCookiesAndRetry(searchUrl, null, SearchUrlReferer, extraHeaders); + + var results = response.Content; + try + { + CQ dom = results; + + var rows = dom["tr"]; + foreach (var row in rows.Skip(1)) + { + var release = new ReleaseInfo(); + var qRow = row.Cq(); + var qTitleLink = qRow.Find("td:eq(1) a:eq(0)").First(); + release.Title = qTitleLink.Find("strong").Text().Trim(); + + // If we search an get no results, we still get a table just with no info. + if (string.IsNullOrWhiteSpace(release.Title)) + { + break; + } + + release.Description = release.Title; + release.Guid = new Uri(qTitleLink.Attr("href")); + release.Comments = release.Guid; + + var dateString = qRow.Find("td:eq(4)").Text(); + release.PublishDate = DateTime.ParseExact(dateString, "dd MMM yy", CultureInfo.InvariantCulture); + + var qLink = qRow.Find("td:eq(2) a"); + release.Link = new Uri(qLink.Attr("href")); + + var sizeStr = qRow.Find("td:eq(5)").Text(); + release.Size = ReleaseInfo.GetBytes(sizeStr); + + var connections = qRow.Find("td:eq(7)").Text().Trim().Split("/".ToCharArray(),StringSplitOptions.RemoveEmptyEntries); + + release.Seeders = ParseUtil.CoerceInt(connections[0].Trim()); + release.Peers = ParseUtil.CoerceInt(connections[1].Trim()) + release.Seeders; + + var rCat = row.Cq().Find("td:eq(0) a").First().Attr("href"); + var rCatIdx = rCat.IndexOf("cat="); + if (rCatIdx > -1) + { + rCat = rCat.Substring(rCatIdx + 4); + } + + release.Category = MapTrackerCatToNewznab(rCat); + + releases.Add(release); + } + } + catch (Exception ex) + { + OnParseError(results, ex); + } + + return releases; + } + } +} diff --git a/src/Jackett/Indexers/BaseIndexer.cs b/src/Jackett/Indexers/BaseIndexer.cs index 71cf91979..2b7efef21 100644 --- a/src/Jackett/Indexers/BaseIndexer.cs +++ b/src/Jackett/Indexers/BaseIndexer.cs @@ -1,451 +1,452 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Jackett.Models; -using Newtonsoft.Json.Linq; -using NLog; -using Jackett.Services; -using Jackett.Utils; -using Jackett.Utils.Clients; -using AutoMapper; -using System.Threading; -using Jackett.Models.IndexerConfig; - -namespace Jackett.Indexers -{ - public abstract class BaseIndexer - { - public string SiteLink { get; private set; } - public string DisplayDescription { get; private set; } - public string DisplayName { get; private set; } - public string ID { get { return GetIndexerID(GetType()); } } - - public bool IsConfigured { get; protected set; } - public TorznabCapabilities TorznabCaps { get; private set; } - protected Logger logger; - protected IIndexerManagerService indexerService; - protected static List cache = new List(); - protected static readonly TimeSpan cacheTime = new TimeSpan(0, 9, 0); - protected IWebClient webclient; - protected IProtectionService protectionService; - protected readonly string downloadUrlBase = ""; - - protected string CookieHeader - { - get { return configData.CookieHeader.Value; } - set { configData.CookieHeader.Value = value; } - } - - - - protected ConfigurationData configData; - - private List categoryMapping = new List(); - - public BaseIndexer(string name, string link, string description, IIndexerManagerService manager, IWebClient client, Logger logger, ConfigurationData configData, IProtectionService p, TorznabCapabilities caps = null, string downloadBase = null) - { - if (!link.EndsWith("/")) - throw new Exception("Site link must end with a slash."); - - DisplayName = name; - DisplayDescription = description; - SiteLink = link; - this.logger = logger; - indexerService = manager; - webclient = client; - protectionService = p; - this.downloadUrlBase = downloadBase; - - this.configData = configData; - - if (caps == null) - caps = TorznabUtil.CreateDefaultTorznabTVCaps(); - TorznabCaps = caps; - - } - - public IEnumerable CleanLinks(IEnumerable releases) - { - if (string.IsNullOrEmpty(downloadUrlBase)) - return releases; - foreach (var release in releases) - { - if (release.Link.ToString().StartsWith(downloadUrlBase)) - { - release.Link = new Uri(release.Link.ToString().Substring(downloadUrlBase.Length), UriKind.Relative); - } - } - - return releases; - } - - public Uri UncleanLink(Uri link) - { - return new Uri(downloadUrlBase + link.ToString(), UriKind.RelativeOrAbsolute); - } - - protected int MapTrackerCatToNewznab(string input) - { - if (null != input) - { - input = input.ToLowerInvariant(); - var mapping = categoryMapping.Where(m => m.TrackerCategory.ToLowerInvariant() == input.ToLowerInvariant()).FirstOrDefault(); - if (mapping != null) - { - return mapping.NewzNabCategory; - } - } - return 0; - } - - public static string GetIndexerID(Type type) - { - return StringUtil.StripNonAlphaNumeric(type.Name.ToLowerInvariant()); - } - - public virtual Task GetConfigurationForSetup() - { - return Task.FromResult(configData); - } - - public virtual void ResetBaseConfig() - { - CookieHeader = string.Empty; - IsConfigured = false; - } - - protected virtual void SaveConfig() - { - indexerService.SaveConfig(this as IIndexer, configData.ToJson(protectionService, forDisplay: false)); - } - - protected void OnParseError(string results, Exception ex) - { - var fileName = string.Format("Error on {0} for {1}.txt", DateTime.Now.ToString("yyyyMMddHHmmss"), DisplayName); - var spacing = string.Join("", Enumerable.Repeat(Environment.NewLine, 5)); - var fileContents = string.Format("{0}{1}{2}", ex, spacing, results); - logger.Error(fileName + fileContents); - } - - protected void CleanCache() - { - foreach (var expired in cache.Where(i => i.Created - DateTime.Now > cacheTime).ToList()) - { - cache.Remove(expired); - } - } - - protected async Task FollowIfRedirect(WebClientStringResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null) - { - var byteResult = new WebClientByteResult(); - // Map to byte - Mapper.Map(response, byteResult); - await FollowIfRedirect(byteResult, referrer, overrideRedirectUrl, overrideCookies); - // Map to string - Mapper.Map(byteResult, response); - } - - protected async Task FollowIfRedirect(WebClientByteResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null) - { - // Follow up to 5 redirects - for (int i = 0; i < 5; i++) - { - if (!response.IsRedirect) - break; - await DoFollowIfRedirect(response, referrer, overrideRedirectUrl, overrideCookies); - } - } - - private async Task DoFollowIfRedirect(WebClientByteResult incomingResponse, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null) - { - if (incomingResponse.IsRedirect) - { - // Do redirect - var redirectedResponse = await webclient.GetBytes(new WebRequest() - { - Url = overrideRedirectUrl ?? incomingResponse.RedirectingTo, - Referer = referrer, - Cookies = overrideCookies ?? CookieHeader - }); - Mapper.Map(redirectedResponse, incomingResponse); - } - } - - - protected void LoadLegacyCookieConfig(JToken jsonConfig) - { - string legacyCookieHeader = (string)jsonConfig["cookie_header"]; - if (!string.IsNullOrEmpty(legacyCookieHeader)) - { - CookieHeader = legacyCookieHeader; - } - else - { - // Legacy cookie key - var jcookies = jsonConfig["cookies"]; - if (jcookies is JArray) - { - var array = (JArray)jcookies; - legacyCookieHeader = string.Empty; - for (int i = 0; i < array.Count; i++) - { - if (i != 0) - legacyCookieHeader += "; "; - legacyCookieHeader += array[i]; - } - CookieHeader = legacyCookieHeader; - } - else if (jcookies != null) - { - CookieHeader = (string)jcookies; - } - } - } - - public virtual void LoadFromSavedConfiguration(JToken jsonConfig) - { - if (jsonConfig is JArray) - { - configData.LoadValuesFromJson(jsonConfig, protectionService); - IsConfigured = true; - } - // read and upgrade old settings file format - else if (jsonConfig is Object) - { - LoadLegacyCookieConfig(jsonConfig); - SaveConfig(); - IsConfigured = true; - } - } - - public async virtual Task Download(Uri link) - { - var response = await RequestBytesWithCookiesAndRetry(link.ToString()); - return response.Content; - } - - protected async Task RequestBytesWithCookiesAndRetry(string url, string cookieOverride = null) - { - Exception lastException = null; - for (int i = 0; i < 3; i++) - { - try - { - return await RequestBytesWithCookies(url, cookieOverride); - } - catch (Exception e) - { - logger.Error(string.Format("On attempt {0} downloading from {1}: {2}", (i + 1), DisplayName, e.Message)); - lastException = e; - } - await Task.Delay(500); - } - - throw lastException; - } - - protected async Task RequestStringWithCookies(string url, string cookieOverride = null, string referer = null) - { - var request = new Utils.Clients.WebRequest() - { - Url = url, - Type = RequestType.GET, - Cookies = CookieHeader, - Referer = referer - }; - - if (cookieOverride != null) - request.Cookies = cookieOverride; - return await webclient.GetString(request); - } - - protected async Task RequestStringWithCookiesAndRetry(string url, string cookieOverride = null, string referer = null) - { - Exception lastException = null; - for (int i = 0; i < 3; i++) - { - try - { - return await RequestStringWithCookies(url, cookieOverride, referer); - } - catch (Exception e) - { - logger.Error(string.Format("On attempt {0} checking for results from {1}: {2}", (i + 1), DisplayName, e.Message)); - lastException = e; - } - await Task.Delay(500); - } - - throw lastException; - } - - protected async Task RequestBytesWithCookies(string url, string cookieOverride = null) - { - var request = new Utils.Clients.WebRequest() - { - Url = url, - Type = RequestType.GET, - Cookies = cookieOverride ?? CookieHeader - }; - - if (cookieOverride != null) - request.Cookies = cookieOverride; - return await webclient.GetBytes(request); - } - - protected async Task PostDataWithCookies(string url, IEnumerable> data, string cookieOverride = null) - { - var request = new Utils.Clients.WebRequest() - { - Url = url, - Type = RequestType.POST, - Cookies = cookieOverride ?? CookieHeader, - PostData = data - }; - return await webclient.GetString(request); - } - - protected async Task PostDataWithCookiesAndRetry(string url, IEnumerable> data, string cookieOverride = null) - { - Exception lastException = null; - for (int i = 0; i < 3; i++) - { - try - { - return await PostDataWithCookies(url, data, cookieOverride); - } - catch (Exception e) - { - logger.Error(string.Format("On attempt {0} checking for results from {1}: {2}", (i + 1), DisplayName, e.Message)); - lastException = e; - } - await Task.Delay(500); - } - - throw lastException; - } - - protected async Task RequestLoginAndFollowRedirect(string url, IEnumerable> data, string cookies, bool returnCookiesFromFirstCall, string redirectUrlOverride = null, string referer = null) - { - var request = new Utils.Clients.WebRequest() - { - Url = url, - Type = RequestType.POST, - Cookies = cookies, - Referer = referer, - PostData = data - }; - var response = await webclient.GetString(request); - var firstCallCookies = response.Cookies; - - if (response.IsRedirect) - { - await FollowIfRedirect(response, request.Url, null, response.Cookies); - } - - if (returnCookiesFromFirstCall) - { - response.Cookies = firstCallCookies; - } - - return response; - } - - protected async Task ConfigureIfOK(string cookies, bool isLoggedin, Func onError) - { - if (isLoggedin) - { - CookieHeader = cookies; - SaveConfig(); - IsConfigured = true; - } - else - { - await onError(); - } - } - - public virtual IEnumerable FilterResults(TorznabQuery query, IEnumerable results) - { - foreach (var result in results) - { - if (query.Categories.Length == 0 || query.Categories.Contains(result.Category) || result.Category == 0 || TorznabCatType.QueryContainsParentCategory(query.Categories, result.Category)) - { - yield return result; - } - } - } - - protected void AddCategoryMapping(string trackerCategory, int newznabCategory) - { - categoryMapping.Add(new CategoryMapping(trackerCategory, newznabCategory)); - } - - protected void AddCategoryMapping(int trackerCategory, TorznabCategory newznabCategory) - { - categoryMapping.Add(new CategoryMapping(trackerCategory.ToString(), newznabCategory.ID)); - if (!TorznabCaps.Categories.Contains(newznabCategory)) - TorznabCaps.Categories.Add(newznabCategory); - } - - protected void AddCategoryMapping(string trackerCategory, TorznabCategory newznabCategory) - { - categoryMapping.Add(new CategoryMapping(trackerCategory.ToString(), newznabCategory.ID)); - if (!TorznabCaps.Categories.Contains(newznabCategory)) - TorznabCaps.Categories.Add(newznabCategory); - } - - protected void AddCategoryMapping(int trackerCategory, int newznabCategory) - { - categoryMapping.Add(new CategoryMapping(trackerCategory.ToString(), newznabCategory)); - } - - protected void AddMultiCategoryMapping(TorznabCategory newznabCategory, params int[] trackerCategories) - { - foreach (var trackerCat in trackerCategories) - { - categoryMapping.Add(new CategoryMapping(trackerCat.ToString(), newznabCategory.ID)); - } - } - - protected void AddMultiCategoryMapping(int trackerCategory, params TorznabCategory[] newznabCategories) - { - foreach (var newznabCat in newznabCategories) - { - categoryMapping.Add(new CategoryMapping(trackerCategory.ToString(), newznabCat.ID)); - } - } - - protected List MapTorznabCapsToTrackers(TorznabQuery query, bool mapChildrenCatsToParent = false) - { - var result = new List(); - foreach (var cat in query.Categories) - { - var queryCats = new List { cat }; - var newznabCat = TorznabCatType.AllCats.FirstOrDefault(c => c.ID == cat); - if (newznabCat != null) - { - queryCats.AddRange(newznabCat.SubCategories.Select(c => c.ID)); - } - - if (mapChildrenCatsToParent) - { - var parentNewznabCat = TorznabCatType.AllCats.FirstOrDefault(c => c.SubCategories.Contains(newznabCat)); - if (parentNewznabCat != null) - { - queryCats.Add(parentNewznabCat.ID); - } - } - - foreach (var mapping in categoryMapping.Where(c => queryCats.Contains(c.NewzNabCategory))) - { - result.Add(mapping.TrackerCategory); - } - } - - return result.Distinct().ToList(); - } - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Jackett.Models; +using Newtonsoft.Json.Linq; +using NLog; +using Jackett.Services; +using Jackett.Utils; +using Jackett.Utils.Clients; +using AutoMapper; +using System.Threading; +using Jackett.Models.IndexerConfig; + +namespace Jackett.Indexers +{ + public abstract class BaseIndexer + { + public string SiteLink { get; private set; } + public string DisplayDescription { get; private set; } + public string DisplayName { get; private set; } + public string ID { get { return GetIndexerID(GetType()); } } + + public bool IsConfigured { get; protected set; } + public TorznabCapabilities TorznabCaps { get; private set; } + protected Logger logger; + protected IIndexerManagerService indexerService; + protected static List cache = new List(); + protected static readonly TimeSpan cacheTime = new TimeSpan(0, 9, 0); + protected IWebClient webclient; + protected IProtectionService protectionService; + protected readonly string downloadUrlBase = ""; + + protected string CookieHeader + { + get { return configData.CookieHeader.Value; } + set { configData.CookieHeader.Value = value; } + } + + + + protected ConfigurationData configData; + + private List categoryMapping = new List(); + + public BaseIndexer(string name, string link, string description, IIndexerManagerService manager, IWebClient client, Logger logger, ConfigurationData configData, IProtectionService p, TorznabCapabilities caps = null, string downloadBase = null) + { + if (!link.EndsWith("/")) + throw new Exception("Site link must end with a slash."); + + DisplayName = name; + DisplayDescription = description; + SiteLink = link; + this.logger = logger; + indexerService = manager; + webclient = client; + protectionService = p; + this.downloadUrlBase = downloadBase; + + this.configData = configData; + + if (caps == null) + caps = TorznabUtil.CreateDefaultTorznabTVCaps(); + TorznabCaps = caps; + + } + + public IEnumerable CleanLinks(IEnumerable releases) + { + if (string.IsNullOrEmpty(downloadUrlBase)) + return releases; + foreach (var release in releases) + { + if (release.Link.ToString().StartsWith(downloadUrlBase)) + { + release.Link = new Uri(release.Link.ToString().Substring(downloadUrlBase.Length), UriKind.Relative); + } + } + + return releases; + } + + public Uri UncleanLink(Uri link) + { + return new Uri(downloadUrlBase + link.ToString(), UriKind.RelativeOrAbsolute); + } + + protected int MapTrackerCatToNewznab(string input) + { + if (null != input) + { + input = input.ToLowerInvariant(); + var mapping = categoryMapping.Where(m => m.TrackerCategory.ToLowerInvariant() == input.ToLowerInvariant()).FirstOrDefault(); + if (mapping != null) + { + return mapping.NewzNabCategory; + } + } + return 0; + } + + public static string GetIndexerID(Type type) + { + return StringUtil.StripNonAlphaNumeric(type.Name.ToLowerInvariant()); + } + + public virtual Task GetConfigurationForSetup() + { + return Task.FromResult(configData); + } + + public virtual void ResetBaseConfig() + { + CookieHeader = string.Empty; + IsConfigured = false; + } + + protected virtual void SaveConfig() + { + indexerService.SaveConfig(this as IIndexer, configData.ToJson(protectionService, forDisplay: false)); + } + + protected void OnParseError(string results, Exception ex) + { + var fileName = string.Format("Error on {0} for {1}.txt", DateTime.Now.ToString("yyyyMMddHHmmss"), DisplayName); + var spacing = string.Join("", Enumerable.Repeat(Environment.NewLine, 5)); + var fileContents = string.Format("{0}{1}{2}", ex, spacing, results); + logger.Error(fileName + fileContents); + } + + protected void CleanCache() + { + foreach (var expired in cache.Where(i => i.Created - DateTime.Now > cacheTime).ToList()) + { + cache.Remove(expired); + } + } + + protected async Task FollowIfRedirect(WebClientStringResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null) + { + var byteResult = new WebClientByteResult(); + // Map to byte + Mapper.Map(response, byteResult); + await FollowIfRedirect(byteResult, referrer, overrideRedirectUrl, overrideCookies); + // Map to string + Mapper.Map(byteResult, response); + } + + protected async Task FollowIfRedirect(WebClientByteResult response, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null) + { + // Follow up to 5 redirects + for (int i = 0; i < 5; i++) + { + if (!response.IsRedirect) + break; + await DoFollowIfRedirect(response, referrer, overrideRedirectUrl, overrideCookies); + } + } + + private async Task DoFollowIfRedirect(WebClientByteResult incomingResponse, string referrer = null, string overrideRedirectUrl = null, string overrideCookies = null) + { + if (incomingResponse.IsRedirect) + { + // Do redirect + var redirectedResponse = await webclient.GetBytes(new WebRequest() + { + Url = overrideRedirectUrl ?? incomingResponse.RedirectingTo, + Referer = referrer, + Cookies = overrideCookies ?? CookieHeader + }); + Mapper.Map(redirectedResponse, incomingResponse); + } + } + + + protected void LoadLegacyCookieConfig(JToken jsonConfig) + { + string legacyCookieHeader = (string)jsonConfig["cookie_header"]; + if (!string.IsNullOrEmpty(legacyCookieHeader)) + { + CookieHeader = legacyCookieHeader; + } + else + { + // Legacy cookie key + var jcookies = jsonConfig["cookies"]; + if (jcookies is JArray) + { + var array = (JArray)jcookies; + legacyCookieHeader = string.Empty; + for (int i = 0; i < array.Count; i++) + { + if (i != 0) + legacyCookieHeader += "; "; + legacyCookieHeader += array[i]; + } + CookieHeader = legacyCookieHeader; + } + else if (jcookies != null) + { + CookieHeader = (string)jcookies; + } + } + } + + public virtual void LoadFromSavedConfiguration(JToken jsonConfig) + { + if (jsonConfig is JArray) + { + configData.LoadValuesFromJson(jsonConfig, protectionService); + IsConfigured = true; + } + // read and upgrade old settings file format + else if (jsonConfig is Object) + { + LoadLegacyCookieConfig(jsonConfig); + SaveConfig(); + IsConfigured = true; + } + } + + public async virtual Task Download(Uri link) + { + var response = await RequestBytesWithCookiesAndRetry(link.ToString()); + return response.Content; + } + + protected async Task RequestBytesWithCookiesAndRetry(string url, string cookieOverride = null) + { + Exception lastException = null; + for (int i = 0; i < 3; i++) + { + try + { + return await RequestBytesWithCookies(url, cookieOverride); + } + catch (Exception e) + { + logger.Error(string.Format("On attempt {0} downloading from {1}: {2}", (i + 1), DisplayName, e.Message)); + lastException = e; + } + await Task.Delay(500); + } + + throw lastException; + } + + protected async Task RequestStringWithCookies(string url, string cookieOverride = null, string referer = null, Dictionary headers = null) + { + var request = new Utils.Clients.WebRequest() + { + Url = url, + Type = RequestType.GET, + Cookies = CookieHeader, + Referer = referer, + Headers = headers + }; + + if (cookieOverride != null) + request.Cookies = cookieOverride; + return await webclient.GetString(request); + } + + protected async Task RequestStringWithCookiesAndRetry(string url, string cookieOverride = null, string referer = null, Dictionary headers = null) + { + Exception lastException = null; + for (int i = 0; i < 3; i++) + { + try + { + return await RequestStringWithCookies(url, cookieOverride, referer, headers); + } + catch (Exception e) + { + logger.Error(string.Format("On attempt {0} checking for results from {1}: {2}", (i + 1), DisplayName, e.Message)); + lastException = e; + } + await Task.Delay(500); + } + + throw lastException; + } + + protected async Task RequestBytesWithCookies(string url, string cookieOverride = null) + { + var request = new Utils.Clients.WebRequest() + { + Url = url, + Type = RequestType.GET, + Cookies = cookieOverride ?? CookieHeader + }; + + if (cookieOverride != null) + request.Cookies = cookieOverride; + return await webclient.GetBytes(request); + } + + protected async Task PostDataWithCookies(string url, IEnumerable> data, string cookieOverride = null) + { + var request = new Utils.Clients.WebRequest() + { + Url = url, + Type = RequestType.POST, + Cookies = cookieOverride ?? CookieHeader, + PostData = data + }; + return await webclient.GetString(request); + } + + protected async Task PostDataWithCookiesAndRetry(string url, IEnumerable> data, string cookieOverride = null) + { + Exception lastException = null; + for (int i = 0; i < 3; i++) + { + try + { + return await PostDataWithCookies(url, data, cookieOverride); + } + catch (Exception e) + { + logger.Error(string.Format("On attempt {0} checking for results from {1}: {2}", (i + 1), DisplayName, e.Message)); + lastException = e; + } + await Task.Delay(500); + } + + throw lastException; + } + + protected async Task RequestLoginAndFollowRedirect(string url, IEnumerable> data, string cookies, bool returnCookiesFromFirstCall, string redirectUrlOverride = null, string referer = null) + { + var request = new Utils.Clients.WebRequest() + { + Url = url, + Type = RequestType.POST, + Cookies = cookies, + Referer = referer, + PostData = data + }; + var response = await webclient.GetString(request); + var firstCallCookies = response.Cookies; + + if (response.IsRedirect) + { + await FollowIfRedirect(response, request.Url, null, response.Cookies); + } + + if (returnCookiesFromFirstCall) + { + response.Cookies = firstCallCookies; + } + + return response; + } + + protected async Task ConfigureIfOK(string cookies, bool isLoggedin, Func onError) + { + if (isLoggedin) + { + CookieHeader = cookies; + SaveConfig(); + IsConfigured = true; + } + else + { + await onError(); + } + } + + public virtual IEnumerable FilterResults(TorznabQuery query, IEnumerable results) + { + foreach (var result in results) + { + if (query.Categories.Length == 0 || query.Categories.Contains(result.Category) || result.Category == 0 || TorznabCatType.QueryContainsParentCategory(query.Categories, result.Category)) + { + yield return result; + } + } + } + + protected void AddCategoryMapping(string trackerCategory, int newznabCategory) + { + categoryMapping.Add(new CategoryMapping(trackerCategory, newznabCategory)); + } + + protected void AddCategoryMapping(int trackerCategory, TorznabCategory newznabCategory) + { + categoryMapping.Add(new CategoryMapping(trackerCategory.ToString(), newznabCategory.ID)); + if (!TorznabCaps.Categories.Contains(newznabCategory)) + TorznabCaps.Categories.Add(newznabCategory); + } + + protected void AddCategoryMapping(string trackerCategory, TorznabCategory newznabCategory) + { + categoryMapping.Add(new CategoryMapping(trackerCategory.ToString(), newznabCategory.ID)); + if (!TorznabCaps.Categories.Contains(newznabCategory)) + TorznabCaps.Categories.Add(newznabCategory); + } + + protected void AddCategoryMapping(int trackerCategory, int newznabCategory) + { + categoryMapping.Add(new CategoryMapping(trackerCategory.ToString(), newznabCategory)); + } + + protected void AddMultiCategoryMapping(TorznabCategory newznabCategory, params int[] trackerCategories) + { + foreach (var trackerCat in trackerCategories) + { + categoryMapping.Add(new CategoryMapping(trackerCat.ToString(), newznabCategory.ID)); + } + } + + protected void AddMultiCategoryMapping(int trackerCategory, params TorznabCategory[] newznabCategories) + { + foreach (var newznabCat in newznabCategories) + { + categoryMapping.Add(new CategoryMapping(trackerCategory.ToString(), newznabCat.ID)); + } + } + + protected List MapTorznabCapsToTrackers(TorznabQuery query, bool mapChildrenCatsToParent = false) + { + var result = new List(); + foreach (var cat in query.Categories) + { + var queryCats = new List { cat }; + var newznabCat = TorznabCatType.AllCats.FirstOrDefault(c => c.ID == cat); + if (newznabCat != null) + { + queryCats.AddRange(newznabCat.SubCategories.Select(c => c.ID)); + } + + if (mapChildrenCatsToParent) + { + var parentNewznabCat = TorznabCatType.AllCats.FirstOrDefault(c => c.SubCategories.Contains(newznabCat)); + if (parentNewznabCat != null) + { + queryCats.Add(parentNewznabCat.ID); + } + } + + foreach (var mapping in categoryMapping.Where(c => queryCats.Contains(c.NewzNabCategory))) + { + result.Add(mapping.TrackerCategory); + } + } + + return result.Distinct().ToList(); + } + } +} diff --git a/src/Jackett/Jackett.csproj b/src/Jackett/Jackett.csproj index b84a20994..71a193b74 100644 --- a/src/Jackett/Jackett.csproj +++ b/src/Jackett/Jackett.csproj @@ -1,609 +1,613 @@ - - - - - Debug - AnyCPU - {E636D5F8-68B4-4903-B4ED-CCFD9C9E899F} - Library - Properties - Jackett - Jackett - v4.5 - 512 - - publish\ - true - Disk - false - Foreground - 7 - Days - false - false - true - 0 - 1.0.0.%2a - false - false - true - - - - - AnyCPU - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - - - AnyCPU - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - - - - - - - - - - - - ..\packages\Autofac.3.5.2\lib\net40\Autofac.dll - True - - - ..\packages\Autofac.Owin.3.1.0\lib\net45\Autofac.Integration.Owin.dll - True - - - ..\packages\Autofac.WebApi2.3.4.0\lib\net45\Autofac.Integration.WebApi.dll - True - - - ..\packages\Autofac.WebApi2.Owin.3.2.0\lib\net45\Autofac.Integration.WebApi.Owin.dll - True - - - ..\packages\AutoMapper.4.0.4\lib\net45\AutoMapper.dll - True - - - ..\packages\CsQuery.1.3.4\lib\net40\CsQuery.dll - True - - - ..\packages\Microsoft.AspNet.Identity.Core.2.2.1\lib\net45\Microsoft.AspNet.Identity.Core.dll - True - - - ..\packages\Microsoft.Owin.3.0.1\lib\net45\Microsoft.Owin.dll - True - - - ..\packages\Microsoft.Owin.FileSystems.3.0.1\lib\net45\Microsoft.Owin.FileSystems.dll - True - - - ..\packages\Microsoft.Owin.Host.HttpListener.3.0.1\lib\net45\Microsoft.Owin.Host.HttpListener.dll - True - - - ..\packages\Microsoft.Owin.Host.SystemWeb.3.0.1\lib\net45\Microsoft.Owin.Host.SystemWeb.dll - True - - - ..\packages\Microsoft.Owin.Hosting.3.0.1\lib\net45\Microsoft.Owin.Hosting.dll - True - - - ..\packages\Microsoft.Owin.StaticFiles.3.0.1\lib\net45\Microsoft.Owin.StaticFiles.dll - True - - - ..\packages\Newtonsoft.Json.7.0.1\lib\net45\Newtonsoft.Json.dll - True - - - ..\packages\NLog.4.0.1\lib\net45\NLog.dll - True - - - ..\packages\NLog.Windows.Forms.2.0.0.0\lib\net35\NLog.Windows.Forms.dll - True - - - ..\packages\Owin.1.0\lib\net40\Owin.dll - True - - - - - - - - ..\packages\Microsoft.Net.Http.2.2.29\lib\net45\System.Net.Http.Extensions.dll - True - - - ..\packages\Microsoft.AspNet.WebApi.Client.5.2.3\lib\net45\System.Net.Http.Formatting.dll - True - - - ..\packages\Microsoft.Net.Http.2.2.29\lib\net45\System.Net.Http.Primitives.dll - True - - - - - - - ..\packages\Microsoft.AspNet.WebApi.Core.5.2.3\lib\net45\System.Web.Http.dll - True - - - ..\packages\Microsoft.AspNet.WebApi.Owin.5.2.3\lib\net45\System.Web.Http.Owin.dll - True - - - ..\packages\Microsoft.AspNet.WebApi.Tracing.5.2.3\lib\net45\System.Web.Http.Tracing.dll - True - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - True - True - TorznabCatType.tt - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - True - True - Resources.resx - - - - - - - - - - - - - - - - - - - - - - Designer - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - - - Always - - - Designer - - - - - - ResXFileCodeGenerator - Resources.Designer.cs - - - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - TextTemplatingFileGenerator - TorznabCatType.generated.cs - - - - PreserveNewest - - - - - {74420A79-CC16-442C-8B1E-7C1B913844F0} - CurlSharp - - - - - False - Microsoft .NET Framework 4.5.1 %28x86 and x64%29 - true - - - False - .NET Framework 3.5 SP1 Client Profile - false - - - False - .NET Framework 3.5 SP1 - false - - - - - - + + + + + Debug + AnyCPU + {E636D5F8-68B4-4903-B4ED-CCFD9C9E899F} + Library + Properties + Jackett + Jackett + v4.5 + 512 + + publish\ + true + Disk + false + Foreground + 7 + Days + false + false + true + 0 + 1.0.0.%2a + false + false + true + + + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + ..\packages\Autofac.3.5.2\lib\net40\Autofac.dll + True + + + ..\packages\Autofac.Owin.3.1.0\lib\net45\Autofac.Integration.Owin.dll + True + + + ..\packages\Autofac.WebApi2.3.4.0\lib\net45\Autofac.Integration.WebApi.dll + True + + + ..\packages\Autofac.WebApi2.Owin.3.2.0\lib\net45\Autofac.Integration.WebApi.Owin.dll + True + + + ..\packages\AutoMapper.4.0.4\lib\net45\AutoMapper.dll + True + + + ..\packages\CsQuery.1.3.4\lib\net40\CsQuery.dll + True + + + ..\packages\Microsoft.AspNet.Identity.Core.2.2.1\lib\net45\Microsoft.AspNet.Identity.Core.dll + True + + + ..\packages\Microsoft.Owin.3.0.1\lib\net45\Microsoft.Owin.dll + True + + + ..\packages\Microsoft.Owin.FileSystems.3.0.1\lib\net45\Microsoft.Owin.FileSystems.dll + True + + + ..\packages\Microsoft.Owin.Host.HttpListener.3.0.1\lib\net45\Microsoft.Owin.Host.HttpListener.dll + True + + + ..\packages\Microsoft.Owin.Host.SystemWeb.3.0.1\lib\net45\Microsoft.Owin.Host.SystemWeb.dll + True + + + ..\packages\Microsoft.Owin.Hosting.3.0.1\lib\net45\Microsoft.Owin.Hosting.dll + True + + + ..\packages\Microsoft.Owin.StaticFiles.3.0.1\lib\net45\Microsoft.Owin.StaticFiles.dll + True + + + ..\packages\Newtonsoft.Json.7.0.1\lib\net45\Newtonsoft.Json.dll + True + + + ..\packages\NLog.4.0.1\lib\net45\NLog.dll + True + + + ..\packages\NLog.Windows.Forms.2.0.0.0\lib\net35\NLog.Windows.Forms.dll + True + + + ..\packages\Owin.1.0\lib\net40\Owin.dll + True + + + + + + + + ..\packages\Microsoft.Net.Http.2.2.29\lib\net45\System.Net.Http.Extensions.dll + True + + + ..\packages\Microsoft.AspNet.WebApi.Client.5.2.3\lib\net45\System.Net.Http.Formatting.dll + True + + + ..\packages\Microsoft.Net.Http.2.2.29\lib\net45\System.Net.Http.Primitives.dll + True + + + + + + + ..\packages\Microsoft.AspNet.WebApi.Core.5.2.3\lib\net45\System.Web.Http.dll + True + + + ..\packages\Microsoft.AspNet.WebApi.Owin.5.2.3\lib\net45\System.Web.Http.Owin.dll + True + + + ..\packages\Microsoft.AspNet.WebApi.Tracing.5.2.3\lib\net45\System.Web.Http.Tracing.dll + True + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + True + True + TorznabCatType.tt + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + True + True + Resources.resx + + + + + + + + + + + + + + + + + + + + + + Designer + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + + + Always + + + Designer + + + + + + ResXFileCodeGenerator + Resources.Designer.cs + + + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + TextTemplatingFileGenerator + TorznabCatType.generated.cs + + + + PreserveNewest + + + + + {74420A79-CC16-442C-8B1E-7C1B913844F0} + CurlSharp + + + + + False + Microsoft .NET Framework 4.5.1 %28x86 and x64%29 + true + + + False + .NET Framework 3.5 SP1 Client Profile + false + + + False + .NET Framework 3.5 SP1 + false + + + + + + - - - - - - - - - - - - - - This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - - - + --> + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + \ No newline at end of file diff --git a/src/Jackett/JackettModule.cs b/src/Jackett/JackettModule.cs index cccbbe192..988b61238 100644 --- a/src/Jackett/JackettModule.cs +++ b/src/Jackett/JackettModule.cs @@ -1,81 +1,82 @@ -using Autofac; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Autofac.Integration.WebApi; -using Jackett.Indexers; -using Jackett.Utils; -using Jackett.Utils.Clients; -using AutoMapper; -using Jackett.Models; - -namespace Jackett -{ - public class JackettModule : Module - { - protected override void Load(ContainerBuilder builder) - { - // Just register everything! - var thisAssembly = typeof(JackettModule).Assembly; - builder.RegisterAssemblyTypes(thisAssembly).Except().AsImplementedInterfaces().SingleInstance(); - builder.RegisterApiControllers(thisAssembly).InstancePerRequest(); - - // Register the best web client for the platform or the override - switch (Startup.ClientOverride) - { - case "httpclient": - builder.RegisterType().As(); - break; - case "safecurl": - builder.RegisterType().As(); - break; - case "libcurl": - builder.RegisterType().As(); - break; - case "automatic": - default: - if (System.Environment.OSVersion.Platform == PlatformID.Unix) - { - builder.RegisterType().As(); - } - else - { - builder.RegisterType().As(); - } - break; - } - - // Register indexers - foreach (var indexer in thisAssembly.GetTypes() - .Where(p => typeof(IIndexer).IsAssignableFrom(p) && !p.IsInterface) - .ToArray()) - { - builder.RegisterType(indexer).Named(BaseIndexer.GetIndexerID(indexer)); - } - - Mapper.CreateMap().ForMember(x => x.Content, opt => opt.Ignore()).AfterMap((be, str) => - { - str.Content = Encoding.UTF8.GetString(be.Content); - }); - - Mapper.CreateMap().ForMember(x => x.Content, opt => opt.Ignore()).AfterMap((str, be) => - { - if (!string.IsNullOrEmpty(str.Content)) - { - be.Content = Encoding.UTF8.GetBytes(str.Content); - } - }); - - Mapper.CreateMap(); - Mapper.CreateMap(); - Mapper.CreateMap(); - - Mapper.CreateMap().AfterMap((r, t) => - { - t.CategoryDesc = TorznabCatType.GetCatDesc(r.Category); - }); - } - } -} +using Autofac; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Autofac.Integration.WebApi; +using Jackett.Indexers; +using Jackett.Utils; +using Jackett.Utils.Clients; +using AutoMapper; +using Jackett.Models; + +namespace Jackett +{ + public class JackettModule : Module + { + protected override void Load(ContainerBuilder builder) + { + // Just register everything! + var thisAssembly = typeof(JackettModule).Assembly; + builder.RegisterAssemblyTypes(thisAssembly).Except().AsImplementedInterfaces().SingleInstance(); + builder.RegisterApiControllers(thisAssembly).InstancePerRequest(); + builder.RegisterType(); + + // Register the best web client for the platform or the override + switch (Startup.ClientOverride) + { + case "httpclient": + builder.RegisterType().As(); + break; + case "safecurl": + builder.RegisterType().As(); + break; + case "libcurl": + builder.RegisterType().As(); + break; + case "automatic": + default: + if (System.Environment.OSVersion.Platform == PlatformID.Unix) + { + builder.RegisterType().As(); + } + else + { + builder.RegisterType().As(); + } + break; + } + + // Register indexers + foreach (var indexer in thisAssembly.GetTypes() + .Where(p => typeof(IIndexer).IsAssignableFrom(p) && !p.IsInterface) + .ToArray()) + { + builder.RegisterType(indexer).Named(BaseIndexer.GetIndexerID(indexer)); + } + + Mapper.CreateMap().ForMember(x => x.Content, opt => opt.Ignore()).AfterMap((be, str) => + { + str.Content = Encoding.UTF8.GetString(be.Content); + }); + + Mapper.CreateMap().ForMember(x => x.Content, opt => opt.Ignore()).AfterMap((str, be) => + { + if (!string.IsNullOrEmpty(str.Content)) + { + be.Content = Encoding.UTF8.GetBytes(str.Content); + } + }); + + Mapper.CreateMap(); + Mapper.CreateMap(); + Mapper.CreateMap(); + + Mapper.CreateMap().AfterMap((r, t) => + { + t.CategoryDesc = TorznabCatType.GetCatDesc(r.Category); + }); + } + } +} diff --git a/src/Jackett/Utils/Clients/HttpWebClient.cs b/src/Jackett/Utils/Clients/HttpWebClient.cs index 89e7eed01..5387dd932 100644 --- a/src/Jackett/Utils/Clients/HttpWebClient.cs +++ b/src/Jackett/Utils/Clients/HttpWebClient.cs @@ -1,110 +1,118 @@ -using AutoMapper; -using Jackett.Models; -using NLog; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Text; -using System.Threading.Tasks; - -namespace Jackett.Utils.Clients -{ - class HttpWebClient : IWebClient - { - private Logger logger; - - public HttpWebClient(Logger l) - { - logger = l; - } - - - public void Init() - { - } - - public async Task GetBytes(WebRequest request) - { - logger.Debug(string.Format("WindowsWebClient:GetBytes(Url:{0})", request.Url)); - var result = await Run(request); - logger.Debug(string.Format("WindowsWebClient: Returning {0} => {1} bytes", result.Status, (result.Content == null ? "" : result.Content.Length.ToString()))); - return result; - } - - public async Task GetString(WebRequest request) - { - logger.Debug(string.Format("WindowsWebClient:GetString(Url:{0})", request.Url)); - var result = await Run(request); - logger.Debug(string.Format("WindowsWebClient: Returning {0} => {1}", result.Status, (result.Content == null ? "" : Encoding.UTF8.GetString(result.Content)))); - return Mapper.Map(result); - } - - private async Task Run(WebRequest request) - { - var cookies = new CookieContainer(); - if (!string.IsNullOrEmpty(request.Cookies)) - { - var uri = new Uri(request.Url); - foreach (var c in request.Cookies.Split(';')) - { - try - { - cookies.SetCookies(uri, c); - } - catch (CookieException ex) - { - logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message); - } - } - } - - var client = new HttpClient(new HttpClientHandler - { - CookieContainer = cookies, - AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more. - UseCookies = true, - }); - - client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent); - HttpResponseMessage response = null; - - if (request.Type == RequestType.POST) - { - var content = new FormUrlEncodedContent(request.PostData); - response = await client.PostAsync(request.Url, content); - } - else - { - response = await client.GetAsync(request.Url); - } - - var result = new WebClientByteResult(); - result.Content = await response.Content.ReadAsByteArrayAsync(); - if (response.Headers.Location != null) - { - result.RedirectingTo = response.Headers.Location.ToString(); - } - result.Status = response.StatusCode; - - // Compatiblity issue between the cookie format and httpclient - // Pull it out manually ignoring the expiry date then set it manually - // http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer - IEnumerable cookieHeaders; - if (response.Headers.TryGetValues("set-cookie", out cookieHeaders)) - { - var cookieBuilder = new StringBuilder(); - foreach (var c in cookieHeaders) - { - cookieBuilder.AppendFormat("{0} ", c.Substring(0, c.IndexOf(';') + 1)); - } - - result.Cookies = cookieBuilder.ToString().TrimEnd(); - } - - ServerUtil.ResureRedirectIsFullyQualified(request, result); - return result; - } - } -} +using AutoMapper; +using Jackett.Models; +using NLog; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; + +namespace Jackett.Utils.Clients +{ + public class HttpWebClient : IWebClient + { + private Logger logger; + + public HttpWebClient(Logger l) + { + logger = l; + } + + + public void Init() + { + } + + public async Task GetBytes(WebRequest request) + { + logger.Debug(string.Format("WindowsWebClient:GetBytes(Url:{0})", request.Url)); + var result = await Run(request); + logger.Debug(string.Format("WindowsWebClient: Returning {0} => {1} bytes", result.Status, (result.Content == null ? "" : result.Content.Length.ToString()))); + return result; + } + + public async Task GetString(WebRequest request) + { + logger.Debug(string.Format("WindowsWebClient:GetString(Url:{0})", request.Url)); + var result = await Run(request); + logger.Debug(string.Format("WindowsWebClient: Returning {0} => {1}", result.Status, (result.Content == null ? "" : Encoding.UTF8.GetString(result.Content)))); + return Mapper.Map(result); + } + + private async Task Run(WebRequest request) + { + var cookies = new CookieContainer(); + if (!string.IsNullOrEmpty(request.Cookies)) + { + var uri = new Uri(request.Url); + foreach (var c in request.Cookies.Split(';')) + { + try + { + cookies.SetCookies(uri, c); + } + catch (CookieException ex) + { + logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message); + } + } + } + + var client = new HttpClient(new HttpClientHandler + { + CookieContainer = cookies, + AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more. + UseCookies = true, + }); + + client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent); + HttpResponseMessage response = null; + + if (request.Headers != null) + { + foreach (var header in request.Headers) + { + client.DefaultRequestHeaders.Add(header.Key, header.Value); + } + } + + if (request.Type == RequestType.POST) + { + var content = new FormUrlEncodedContent(request.PostData); + response = await client.PostAsync(request.Url, content); + } + else + { + response = await client.GetAsync(request.Url); + } + + var result = new WebClientByteResult(); + result.Content = await response.Content.ReadAsByteArrayAsync(); + if (response.Headers.Location != null) + { + result.RedirectingTo = response.Headers.Location.ToString(); + } + result.Status = response.StatusCode; + + // Compatiblity issue between the cookie format and httpclient + // Pull it out manually ignoring the expiry date then set it manually + // http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer + IEnumerable cookieHeaders; + if (response.Headers.TryGetValues("set-cookie", out cookieHeaders)) + { + var cookieBuilder = new StringBuilder(); + foreach (var c in cookieHeaders) + { + cookieBuilder.AppendFormat("{0} ", c.Substring(0, c.IndexOf(';') + 1)); + } + + result.Cookies = cookieBuilder.ToString().TrimEnd(); + } + + ServerUtil.ResureRedirectIsFullyQualified(request, result); + return result; + } + } +} diff --git a/src/Jackett/Utils/Clients/WebRequest.cs b/src/Jackett/Utils/Clients/WebRequest.cs index 2d8862b18..a9ae22d6e 100644 --- a/src/Jackett/Utils/Clients/WebRequest.cs +++ b/src/Jackett/Utils/Clients/WebRequest.cs @@ -1,89 +1,95 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net.Http; -using System.Text; -using System.Threading.Tasks; - -namespace Jackett.Utils.Clients -{ - public class WebRequest - { - public WebRequest() - { - PostData = new List>(); - Type = RequestType.GET; - } - - public WebRequest(string url) - { - PostData = new List>(); - Type = RequestType.GET; - Url = url; - } - - public string Url { get; set; } - public IEnumerable> PostData { get; set; } - public string Cookies { get; set; } - public string Referer { get; set; } - public RequestType Type { get; set; } - - - public override bool Equals(System.Object obj) - { - if (obj is WebRequest) - { - var other = obj as WebRequest; - var postDataSame = PostData == null && other.PostData == null; - if (!postDataSame) - { - if (!(PostData == null || other.PostData == null)) - { - var ok = PostData.Count() == other.PostData.Count(); - foreach (var i in PostData) - { - if (!other.PostData.Any(item => item.Key == i.Key)) - { - ok = false; - break; - } - - if (PostData.FirstOrDefault(item => item.Key == i.Key).Value != other.PostData.FirstOrDefault(item => item.Key == i.Key).Value) - { - ok = false; - break; - } - } - - if (ok) - { - postDataSame = true; - } - } - } - - return other.Url == Url && - other.Referer == Referer && - other.Cookies == Cookies && - other.Type == Type && - postDataSame; - - } - else - { - return false; - } - } - - public override int GetHashCode() - { - return base.GetHashCode(); - } - } - - public enum RequestType - { - GET, - POST - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; + +namespace Jackett.Utils.Clients +{ + public class WebRequest + { + public WebRequest() + { + PostData = new List>(); + Type = RequestType.GET; + Headers = new Dictionary(); + } + + public WebRequest(string url) + { + PostData = new List>(); + Type = RequestType.GET; + Url = url; + Headers = new Dictionary(); + } + + public string Url { get; set; } + public IEnumerable> PostData { get; set; } + public string Cookies { get; set; } + public string Referer { get; set; } + public RequestType Type { get; set; } + + /// + /// Warning this is only implemented on HTTPWebClient currently! + /// + public Dictionary Headers { get; set; } + + public override bool Equals(System.Object obj) + { + if (obj is WebRequest) + { + var other = obj as WebRequest; + var postDataSame = PostData == null && other.PostData == null; + if (!postDataSame) + { + if (!(PostData == null || other.PostData == null)) + { + var ok = PostData.Count() == other.PostData.Count(); + foreach (var i in PostData) + { + if (!other.PostData.Any(item => item.Key == i.Key)) + { + ok = false; + break; + } + + if (PostData.FirstOrDefault(item => item.Key == i.Key).Value != other.PostData.FirstOrDefault(item => item.Key == i.Key).Value) + { + ok = false; + break; + } + } + + if (ok) + { + postDataSame = true; + } + } + } + + return other.Url == Url && + other.Referer == Referer && + other.Cookies == Cookies && + other.Type == Type && + postDataSame; + + } + else + { + return false; + } + } + + public override int GetHashCode() + { + return base.GetHashCode(); + } + } + + public enum RequestType + { + GET, + POST + } +}