From e660c57ccb25b71f3a7ba27c49ac8946efcaa050 Mon Sep 17 00:00:00 2001 From: KZ Date: Thu, 13 Aug 2015 22:51:49 +0100 Subject: [PATCH] Category mapping on TorrentLeech and AnimeBytes. Fix sparadic issue where downloads fail due to the url being too long. --- src/Jackett.Console/Jackett.Console.csproj | 2 +- src/Jackett/Content/custom.js | 2 +- src/Jackett/Controllers/AdminController.cs | 15 +++- .../Controllers/BlackholeController.cs | 15 ++-- src/Jackett/Controllers/DownloadController.cs | 21 ++++-- src/Jackett/Controllers/PotatoController.cs | 3 +- src/Jackett/Controllers/TorznabController.cs | 5 +- src/Jackett/Indexers/AlphaRatio.cs | 1 + src/Jackett/Indexers/AnimeBytes.cs | 63 +++++++++++++++-- src/Jackett/Indexers/BakaBT.cs | 2 +- src/Jackett/Indexers/BaseIndexer.cs | 26 ++++++- src/Jackett/Indexers/FileList.cs | 2 +- src/Jackett/Indexers/IIndexer.cs | 3 + src/Jackett/Indexers/TorrentLeech.cs | 69 +++++++++++++++++-- src/Jackett/Models/ReleaseInfo.cs | 10 --- src/Jackett/Models/TrackerCacheResult.cs | 1 + src/Jackett/Services/CacheService.cs | 17 ++--- src/Jackett/Services/IndexerManagerService.cs | 1 + src/Jackett/Services/ServerService.cs | 12 ++++ src/Jackett/Startup.cs | 12 ++-- 20 files changed, 222 insertions(+), 60 deletions(-) diff --git a/src/Jackett.Console/Jackett.Console.csproj b/src/Jackett.Console/Jackett.Console.csproj index 5799b313e..3d599c9b7 100644 --- a/src/Jackett.Console/Jackett.Console.csproj +++ b/src/Jackett.Console/Jackett.Console.csproj @@ -17,7 +17,7 @@ - AnyCPU + x86 true full false diff --git a/src/Jackett/Content/custom.js b/src/Jackett/Content/custom.js index 5aec604fe..89e4540c3 100644 --- a/src/Jackett/Content/custom.js +++ b/src/Jackett/Content/custom.js @@ -53,7 +53,7 @@ $("#jackett-show-releases").click(function () { { "pageLength": 20, "lengthMenu": [[10, 20, 50, -1], [10, 20, 50, "All"]], - "order": [[2, "desc"]], + "order": [[0, "desc"]], "columnDefs": [ { "targets": 0, diff --git a/src/Jackett/Controllers/AdminController.cs b/src/Jackett/Controllers/AdminController.cs index 397c51295..174b869bf 100644 --- a/src/Jackett/Controllers/AdminController.cs +++ b/src/Jackett/Controllers/AdminController.cs @@ -396,8 +396,19 @@ namespace Jackett.Controllers [HttpGet] public List GetCache() { - var severUrl = string.Format("{0}://{1}:{2}/", Request.RequestUri.Scheme, Request.RequestUri.Host, Request.RequestUri.Port); - return cacheService.GetCachedResults(severUrl); + var serverUrl = string.Format("{0}://{1}:{2}/", Request.RequestUri.Scheme, Request.RequestUri.Host, Request.RequestUri.Port); + var results = cacheService.GetCachedResults(); + + foreach (var result in results) + { + var link = result.Link; + result.Link = serverService.ConvertToProxyLink(link, serverUrl, result.TrackerId); + if (result.Link != null && result.Link.Scheme != "magnet" && !string.IsNullOrWhiteSpace(Engine.Server.Config.BlackholeDir)) + result.BlackholeLink = serverService.ConvertToProxyLink(link, serverUrl, result.TrackerId, "bh", string.Empty); + + } + + return results; } [Route("GetLogs")] diff --git a/src/Jackett/Controllers/BlackholeController.cs b/src/Jackett/Controllers/BlackholeController.cs index a359d3cfc..24a90b227 100644 --- a/src/Jackett/Controllers/BlackholeController.cs +++ b/src/Jackett/Controllers/BlackholeController.cs @@ -21,15 +21,17 @@ namespace Jackett.Controllers { private Logger logger; private IIndexerManagerService indexerService; + IServerService serverService; - public BlackholeController(IIndexerManagerService i, Logger l) + public BlackholeController(IIndexerManagerService i, Logger l, IServerService s) { logger = l; indexerService = i; + serverService = s; } [HttpGet] - public async Task Blackhole(string indexerID, string path) + public async Task Blackhole(string indexerID, string path, string apikey) { var jsonReply = new JObject(); @@ -42,8 +44,13 @@ namespace Jackett.Controllers throw new Exception("This indexer is not configured."); } - var remoteFile = Encoding.UTF8.GetString(HttpServerUtility.UrlTokenDecode(path)); - var downloadBytes = await indexer.Download(new Uri(remoteFile, UriKind.RelativeOrAbsolute)); + if (serverService.Config.APIKey != apikey) + throw new Exception("Incorrect API key"); + + var remoteFile = new Uri(Encoding.UTF8.GetString(HttpServerUtility.UrlTokenDecode(path)), UriKind.RelativeOrAbsolute); + remoteFile = indexer.UncleanLink(remoteFile); + + var downloadBytes = await indexer.Download(remoteFile); if (string.IsNullOrWhiteSpace(Engine.Server.Config.BlackholeDir)) { diff --git a/src/Jackett/Controllers/DownloadController.cs b/src/Jackett/Controllers/DownloadController.cs index 458d5e276..ee3e8373d 100644 --- a/src/Jackett/Controllers/DownloadController.cs +++ b/src/Jackett/Controllers/DownloadController.cs @@ -17,17 +17,19 @@ namespace Jackett.Controllers [JackettAPINoCache] public class DownloadController : ApiController { - private Logger logger; - private IIndexerManagerService indexerService; + Logger logger; + IIndexerManagerService indexerService; + IServerService serverService; - public DownloadController(IIndexerManagerService i, Logger l) + public DownloadController(IIndexerManagerService i, Logger l, IServerService s) { logger = l; indexerService = i; + serverService = s; } [HttpGet] - public async Task Download(string indexerID, string path) + public async Task Download(string indexerID, string path, string apikey) { try { @@ -39,8 +41,15 @@ namespace Jackett.Controllers return Request.CreateResponse(HttpStatusCode.Forbidden, "This indexer is not configured."); } - var remoteFile = Encoding.UTF8.GetString(HttpServerUtility.UrlTokenDecode(path)); - var downloadBytes = await indexer.Download(new Uri(remoteFile, UriKind.RelativeOrAbsolute)); + path = Encoding.UTF8.GetString(HttpServerUtility.UrlTokenDecode(path)); + + if (serverService.Config.APIKey != apikey) + return new HttpResponseMessage(HttpStatusCode.Unauthorized); + + var target = new Uri(path, UriKind.RelativeOrAbsolute); + target = indexer.UncleanLink(target); + + var downloadBytes = await indexer.Download(target); var result = new HttpResponseMessage(HttpStatusCode.OK); result.Content = new ByteArrayContent(downloadBytes); diff --git a/src/Jackett/Controllers/PotatoController.cs b/src/Jackett/Controllers/PotatoController.cs index 45a85bf10..d8a8a3f2b 100644 --- a/src/Jackett/Controllers/PotatoController.cs +++ b/src/Jackett/Controllers/PotatoController.cs @@ -95,6 +95,7 @@ namespace Jackett.Controllers if (!string.IsNullOrWhiteSpace(torznabQuery.SanitizedSearchTerm)) { releases = await indexer.PerformQuery(torznabQuery); + releases = indexer.CleanLinks(releases); } // Cache non query results @@ -112,7 +113,7 @@ namespace Jackett.Controllers foreach (var r in releases) { var release = Mapper.Map(r); - release.Link = release.ConvertToProxyLink(serverUrl, indexerID); + release.Link = serverService.ConvertToProxyLink(release.Link, serverUrl, indexerID); potatoResponse.results.Add(new TorrentPotatoResponseItem() { diff --git a/src/Jackett/Controllers/TorznabController.cs b/src/Jackett/Controllers/TorznabController.cs index f13bfab8e..0e7e0fa9c 100644 --- a/src/Jackett/Controllers/TorznabController.cs +++ b/src/Jackett/Controllers/TorznabController.cs @@ -64,9 +64,10 @@ namespace Jackett.Controllers } var releases = await indexer.PerformQuery(torznabQuery); + releases = indexer.CleanLinks(releases); // Some trackers do not keep their clocks up to date and can be ~20 minutes out! - foreach(var release in releases) + foreach (var release in releases) { if (release.PublishDate > DateTime.Now) release.PublishDate = DateTime.Now; @@ -115,7 +116,7 @@ namespace Jackett.Controllers foreach(var result in releases) { var clone = Mapper.Map(result); - clone.Link = clone.ConvertToProxyLink(serverUrl, indexerID); + clone.Link = serverService.ConvertToProxyLink(clone.Link, serverUrl, indexerID); resultPage.Releases.Add(clone); } diff --git a/src/Jackett/Indexers/AlphaRatio.cs b/src/Jackett/Indexers/AlphaRatio.cs index 44f5d3cea..d498d26e5 100644 --- a/src/Jackett/Indexers/AlphaRatio.cs +++ b/src/Jackett/Indexers/AlphaRatio.cs @@ -35,6 +35,7 @@ namespace Jackett.Indexers client: w, logger: l, p: ps, + downloadBase: "https://alpharatio.cc/torrents.php?action=download&id=", configData: new ConfigurationDataBasicLogin()) { AddCategoryMapping(1, TorznabCatType.TVSD); diff --git a/src/Jackett/Indexers/AnimeBytes.cs b/src/Jackett/Indexers/AnimeBytes.cs index d88d41438..71bc47b5a 100644 --- a/src/Jackett/Indexers/AnimeBytes.cs +++ b/src/Jackett/Indexers/AnimeBytes.cs @@ -24,7 +24,7 @@ namespace Jackett.Indexers public class AnimeBytes : BaseIndexer, IIndexer { private string LoginUrl { get { return SiteLink + "user/login"; } } - private string SearchUrl { get { return SiteLink + "torrents.php?filter_cat[1]=1"; } } + private string SearchUrl { get { return SiteLink + "torrents.php?"; } } public bool AllowRaws { get { return configData.IncludeRaw.Value; } } new ConfigurationDataAnimeBytes configData @@ -39,11 +39,17 @@ namespace Jackett.Indexers description: "Powered by Tentacles", manager: i, client: client, - caps: new TorznabCapabilities(TorznabCatType.TVAnime), + caps: new TorznabCapabilities(TorznabCatType.TVAnime, + TorznabCatType.Movies, + TorznabCatType.BooksComics, + TorznabCatType.ConsolePSP, + TorznabCatType.ConsoleOther, + TorznabCatType.PCGames), logger: l, p: ps, configData: new ConfigurationDataAnimeBytes()) { + } public async Task ApplyConfiguration(JToken configJson) @@ -125,6 +131,8 @@ namespace Jackett.Indexers public async Task> GetResults(string searchTerm) { + var cleanSearchTerm = HttpUtility.UrlEncode(searchTerm); + // This tracker only deals with full seasons so chop off the episode/season number if we have it D: if (!string.IsNullOrWhiteSpace(searchTerm)) { @@ -151,7 +159,7 @@ namespace Jackett.Indexers // Only include the query bit if its required as hopefully the site caches the non query page if (!string.IsNullOrWhiteSpace(searchTerm)) { - queryUrl += "&action=advanced&search_type=title&sort=time_added&way=desc&anime%5Btv_series%5D=1&searchstr=" + WebUtility.UrlEncode(searchTerm); + queryUrl += string.Format("searchstr={0}&action=advanced&search_type=title&year=&year2=&tags=&tags_type=0&sort=time_added&way=desc&hentai=2&releasegroup=&epcount=&epcount2=&artbooktitle=", cleanSearchTerm); } // Get the content from the tracker @@ -162,7 +170,7 @@ namespace Jackett.Indexers try { var releaseInfo = "S01"; - var root = dom.Find(".anime"); + var root = dom.Find(".group_cont"); // We may have got redirected to the series page if we have none of these if (root.Count() == 0) root = dom.Find(".torrent_table"); @@ -215,7 +223,7 @@ namespace Jackett.Indexers var releaseRows = seriesCq.Find(".torrent_group tr"); // Skip the first two info rows - for (int r = 2; r < releaseRows.Count(); r++) + for (int r = 1; r < releaseRows.Count(); r++) { var row = releaseRows.Get(r); var rowCq = row.Cq(); @@ -257,6 +265,37 @@ namespace Jackett.Indexers release.Guid = new Uri(SiteLink + infoLink.Attributes.GetAttribute("href") + "&nh=" + StringUtil.Hash(title)); // Sonarr should dedupe on this url - allow a url per name. release.Link = new Uri(downloadLink.Attributes.GetAttribute("href"), UriKind.Relative); + var category = seriesCq.Find("a[title=\"View Torrent\"]").Text().Trim(); + if (category == "TV Series") + release.Category = TorznabCatType.TVAnime.ID; + + // Ignore these categories as they'll cause hell with the matcher + // TV Special, OVA, ONA, DVD Special, BD Special + + if (category == "Movie") + release.Category = TorznabCatType.Movies.ID; + + if (category == "Manga" || category == "Oneshot" || category == "Anthology" || category == "Manhwa" || category == "Manhua" || category == "Light Novel") + release.Category = TorznabCatType.BooksComics.ID; + + if (category == "Novel" || category == "Artbook") + release.Category = TorznabCatType.BooksComics.ID; + + if (category == "Game" || category == "Visual Novel") + { + var description = rowCq.Find(".torrent_properties a:eq(1)").Text(); + if (description.Contains(" PSP ")) + release.Category = TorznabCatType.ConsolePSP.ID; + if (description.Contains("PSX")) + release.Category = TorznabCatType.ConsoleOther.ID; + if (description.Contains(" NES ")) + release.Category = TorznabCatType.ConsoleOther.ID; + if (description.Contains(" PC ")) + release.Category = TorznabCatType.PCGames.ID; + } + + + // We dont actually have a release name >.> so try to create one var releaseTags = infoLink.InnerText.Split("|".ToCharArray(), StringSplitOptions.RemoveEmptyEntries).ToList(); for (int i = releaseTags.Count - 1; i >= 0; i--) @@ -287,10 +326,19 @@ namespace Jackett.Indexers for (int i = 0; i + 1 < releaseTags.Count(); i++) { + if (releaseTags[i] == "Raw" && !AllowRaws) + continue; infoString += "[" + releaseTags[i] + "]"; } - release.Title = string.Format("{0}{1} {2} {3}", group, title, releaseInfo, infoString); + if (category == "Movie") + { + release.Title = string.Format("{0} {1} {2}{3}", title, year, group, infoString); + } + else + { + release.Title = string.Format("{0}{1} {2} {3}", group, title, releaseInfo, infoString); + } release.Description = title; var size = rowCq.Find(".torrent_size"); @@ -306,7 +354,8 @@ namespace Jackett.Indexers release.Seeders = ParseUtil.CoerceInt(rowCq.Find(".torrent_seeders").Text()); release.Peers = release.Seeders + ParseUtil.CoerceInt(rowCq.Find(".torrent_leechers").Text()); - releases.Add(release); + if (release.Category != 0) + releases.Add(release); } } } diff --git a/src/Jackett/Indexers/BakaBT.cs b/src/Jackett/Indexers/BakaBT.cs index aa42a3bc4..4112dd18a 100644 --- a/src/Jackett/Indexers/BakaBT.cs +++ b/src/Jackett/Indexers/BakaBT.cs @@ -147,7 +147,7 @@ namespace Jackett.Indexers release.Size = ReleaseInfo.GetBytes(size); //22 Jul 15 - var dateStr = qRow.Find(".added . datetime").First().Text().Replace("'", string.Empty); + var dateStr = qRow.Find(".added").First().Text().Replace("'", string.Empty); if (dateStr.Split(' ')[0].Length == 1) dateStr = "0" + dateStr; diff --git a/src/Jackett/Indexers/BaseIndexer.cs b/src/Jackett/Indexers/BaseIndexer.cs index 3d1259a50..dd528f4b6 100644 --- a/src/Jackett/Indexers/BaseIndexer.cs +++ b/src/Jackett/Indexers/BaseIndexer.cs @@ -30,6 +30,7 @@ namespace Jackett.Indexers protected static readonly TimeSpan cacheTime = new TimeSpan(0, 9, 0); protected IWebClient webclient; protected IProtectionService protectionService; + protected readonly string downloadUrlBase = ""; protected string CookieHeader { @@ -37,11 +38,13 @@ namespace Jackett.Indexers set { configData.CookieHeader.Value = value; } } + + protected ConfigurationData configData; private List categoryMapping = new List(); - public BaseIndexer(string name, string link, string description, IIndexerManagerService manager, IWebClient client, Logger logger, ConfigurationData configData, IProtectionService p, TorznabCapabilities caps = null) + public BaseIndexer(string name, string link, string description, IIndexerManagerService manager, IWebClient client, Logger logger, ConfigurationData configData, IProtectionService p, TorznabCapabilities caps = null, string downloadBase = null) { if (!link.EndsWith("/")) throw new Exception("Site link must end with a slash."); @@ -53,6 +56,7 @@ namespace Jackett.Indexers indexerService = manager; webclient = client; protectionService = p; + this.downloadUrlBase = downloadBase; this.configData = configData; @@ -62,6 +66,26 @@ namespace Jackett.Indexers } + public IEnumerable CleanLinks(IEnumerable releases) + { + if (string.IsNullOrEmpty(downloadUrlBase)) + return releases; + foreach(var release in releases) + { + if (release.Link.ToString().StartsWith(downloadUrlBase)) + { + release.Link = new Uri(release.Link.ToString().Substring(downloadUrlBase.Length), UriKind.Relative); + } + } + + return releases; + } + + public Uri UncleanLink(Uri link) + { + return new Uri(downloadUrlBase + link.ToString(), UriKind.RelativeOrAbsolute); + } + protected int MapTrackerCatToNewznab(string input) { if (null != input) diff --git a/src/Jackett/Indexers/FileList.cs b/src/Jackett/Indexers/FileList.cs index acfc371ae..4ea0c6589 100644 --- a/src/Jackett/Indexers/FileList.cs +++ b/src/Jackett/Indexers/FileList.cs @@ -110,7 +110,7 @@ namespace Jackett.Indexers //22:05:3716/02/2013 var dateStr = qRow.Find(".torrenttable:eq(5)").Text().Trim(); - release.PublishDate = DateTime.ParseExact(dateStr, "H:mm:ssdd/MM/yyyy", CultureInfo.InvariantCulture); + release.PublishDate = DateTime.ParseExact(dateStr, "H:mm:ssdd/MM/yyyy", CultureInfo.InvariantCulture).AddHours(-2); var qLink = qRow.Find(".torrenttable:eq(2) a").First(); release.Link = new Uri(SiteLink + qLink.Attr("href")); diff --git a/src/Jackett/Indexers/IIndexer.cs b/src/Jackett/Indexers/IIndexer.cs index 662dd8ebe..e3dd527dd 100644 --- a/src/Jackett/Indexers/IIndexer.cs +++ b/src/Jackett/Indexers/IIndexer.cs @@ -37,5 +37,8 @@ namespace Jackett.Indexers IEnumerable FilterResults(TorznabQuery query, IEnumerable input); Task Download(Uri link); + + IEnumerable CleanLinks(IEnumerable releases); + Uri UncleanLink(Uri link); } } diff --git a/src/Jackett/Indexers/TorrentLeech.cs b/src/Jackett/Indexers/TorrentLeech.cs index 280276059..e6dedc191 100644 --- a/src/Jackett/Indexers/TorrentLeech.cs +++ b/src/Jackett/Indexers/TorrentLeech.cs @@ -21,7 +21,7 @@ namespace Jackett.Indexers public class TorrentLeech : BaseIndexer, IIndexer { private string LoginUrl { get { return SiteLink + "user/account/login/"; } } - private string SearchUrl { get { return SiteLink + "torrents/browse/index/query/{0}/categories/2%2C26%2C27%2C32/orderby/added?"; } } + private string SearchUrl { get { return SiteLink + "torrents/browse/index/"; } } new ConfigurationDataBasicLogin configData { @@ -38,8 +38,45 @@ namespace Jackett.Indexers client: wc, logger: l, p: ps, + downloadBase: "http://www.torrentleech.org/download/", configData: new ConfigurationDataBasicLogin()) { + + AddCategoryMapping(8, TorznabCatType.MoviesSD); // cam + AddCategoryMapping(9, TorznabCatType.MoviesSD); //ts + AddCategoryMapping(10, TorznabCatType.MoviesSD); // Sceener + AddCategoryMapping(11, TorznabCatType.MoviesSD); + AddCategoryMapping(12, TorznabCatType.MoviesSD); + AddCategoryMapping(13, TorznabCatType.MoviesHD); + AddCategoryMapping(14, TorznabCatType.MoviesHD); + AddCategoryMapping(15, TorznabCatType.Movies); // Boxsets + AddCategoryMapping(29, TorznabCatType.TVDocumentary); + + AddCategoryMapping(26, TorznabCatType.TVSD); + AddCategoryMapping(27, TorznabCatType.TV); // Boxsets + AddCategoryMapping(32, TorznabCatType.TVHD); + + AddCategoryMapping(17, TorznabCatType.PCGames); + AddCategoryMapping(18, TorznabCatType.ConsoleXbox); + AddCategoryMapping(19, TorznabCatType.ConsoleXbox360); + // 20 PS2 + AddCategoryMapping(21, TorznabCatType.ConsolePS3); + AddCategoryMapping(22, TorznabCatType.ConsolePSP); + AddCategoryMapping(28, TorznabCatType.ConsoleWii); + AddCategoryMapping(30, TorznabCatType.ConsoleNDS); + + AddCategoryMapping(16, TorznabCatType.AudioVideo); + AddCategoryMapping(31, TorznabCatType.Audio); + + AddCategoryMapping(34, TorznabCatType.TVAnime); + AddCategoryMapping(35, TorznabCatType.TV); // Cartoons + + AddCategoryMapping(5, TorznabCatType.Books); + + AddCategoryMapping(23, TorznabCatType.PCISO); + AddCategoryMapping(24, TorznabCatType.PCMac); + AddCategoryMapping(25, TorznabCatType.PCPhoneOther); + AddCategoryMapping(33, TorznabCatType.PC0day); } public async Task ApplyConfiguration(JToken configJson) @@ -66,8 +103,27 @@ namespace Jackett.Indexers { var releases = new List(); var searchString = query.SanitizedSearchTerm + " " + query.GetEpisodeSearchString(); - var episodeSearchUrl = string.Format(SearchUrl, HttpUtility.UrlEncode(searchString)); - var results = await RequestStringWithCookiesAndRetry(episodeSearchUrl); + var searchUrl = SearchUrl; + + if (!string.IsNullOrWhiteSpace(searchString)) + { + searchUrl += "query/" + HttpUtility.UrlEncode(searchString) + "/"; + } + string.Format(SearchUrl, HttpUtility.UrlEncode(searchString)); + + var cats = MapTorznabCapsToTrackers(query); + if (cats.Count > 0) + { + searchUrl += "categories/"; + foreach (var cat in cats) + { + if (!searchUrl.EndsWith("/")) + searchUrl += ","; + searchUrl += cat; + } + } + + var results = await RequestStringWithCookiesAndRetry(searchUrl); try { CQ dom = results.Content; @@ -86,12 +142,12 @@ namespace Jackett.Indexers release.MinimumSeedTime = 172800; CQ qLink = qRow.Find(".title > a").First(); - release.Guid = new Uri(SiteLink + qLink.Attr("href")); + release.Guid = new Uri(SiteLink + qLink.Attr("href").Substring(1)); release.Comments = release.Guid; release.Title = qLink.Text(); release.Description = release.Title; - release.Link = new Uri(SiteLink + qRow.Find(".quickdownload > a").Attr("href")); + release.Link = new Uri(SiteLink + qRow.Find(".quickdownload > a").Attr("href").Substring(1)); var dateString = qRow.Find(".name")[0].InnerText.Trim().Replace(" ", string.Empty).Replace("Addedinon", string.Empty); //"2015-04-25 23:38:12" @@ -104,6 +160,9 @@ namespace Jackett.Indexers release.Seeders = ParseUtil.CoerceInt(qRow.Find(".seeders").Text()); release.Peers = release.Seeders + ParseUtil.CoerceInt(qRow.Find(".leechers").Text()); + var category = qRow.Find(".category a").Attr("href").Replace("/torrents/browse/index/categories/",string.Empty); + release.Category = MapTrackerCatToNewznab(category); + releases.Add(release); } } diff --git a/src/Jackett/Models/ReleaseInfo.cs b/src/Jackett/Models/ReleaseInfo.cs index b4dd95d19..4c3318075 100644 --- a/src/Jackett/Models/ReleaseInfo.cs +++ b/src/Jackett/Models/ReleaseInfo.cs @@ -91,15 +91,5 @@ namespace Jackett.Models { return (long)(kb * 1024f); } - - public Uri ConvertToProxyLink(string serverUrl, string indexerId, string action = "download") - { - if (Link == null || (Link.IsAbsoluteUri && Link.Scheme == "magnet")) - return Link; - var originalLink = Link; - var encodedLink = HttpServerUtility.UrlTokenEncode(Encoding.UTF8.GetBytes(originalLink.ToString())) + "/t.torrent"; - var proxyLink = string.Format("{0}api/{1}/{2}/{3}", serverUrl, indexerId, action, encodedLink); - return new Uri(proxyLink); - } } } diff --git a/src/Jackett/Models/TrackerCacheResult.cs b/src/Jackett/Models/TrackerCacheResult.cs index 1475c4ac8..01a7b4368 100644 --- a/src/Jackett/Models/TrackerCacheResult.cs +++ b/src/Jackett/Models/TrackerCacheResult.cs @@ -10,6 +10,7 @@ namespace Jackett.Models { public DateTime FirstSeen { get; set; } public string Tracker { get; set; } + public string TrackerId { get; set; } public string CategoryDesc { get; set; } public Uri BlackholeLink { get; set; } } diff --git a/src/Jackett/Services/CacheService.cs b/src/Jackett/Services/CacheService.cs index 536c68620..b9c0f0ccf 100644 --- a/src/Jackett/Services/CacheService.cs +++ b/src/Jackett/Services/CacheService.cs @@ -12,7 +12,7 @@ namespace Jackett.Services public interface ICacheService { void CacheRssResults(IIndexer indexer, IEnumerable releases); - List GetCachedResults(string serverUrl); + List GetCachedResults(); int GetNewItemCount(IIndexer indexer, IEnumerable releases); } @@ -37,12 +37,6 @@ namespace Jackett.Services foreach(var release in releases.OrderByDescending(i=>i.PublishDate)) { - // Skip old releases - if(release.PublishDate-DateTime.Now> AGE_LIMIT) - { - continue; - } - var existingItem = trackerCache.Results.Where(i => i.Result.Guid == release.Guid).FirstOrDefault(); if (existingItem == null) { @@ -86,7 +80,7 @@ namespace Jackett.Services } } - public List GetCachedResults(string serverUrl) + public List GetCachedResults() { lock (cache) { @@ -94,16 +88,13 @@ namespace Jackett.Services foreach(var tracker in cache) { - foreach(var release in tracker.Results) + foreach(var release in tracker.Results.OrderByDescending(i => i.Result.PublishDate).Take(300)) { var item = Mapper.Map(release.Result); item.FirstSeen = release.Created; item.Tracker = tracker.TrackerName; + item.TrackerId = tracker.TrackerId; item.Peers = item.Peers - item.Seeders; // Use peers as leechers - item.Link = item.ConvertToProxyLink(serverUrl, tracker.TrackerId); - if(item.Link!=null && item.Link.Scheme != "magnet" && !string.IsNullOrWhiteSpace(Engine.Server.Config.BlackholeDir)) - item.BlackholeLink = item.ConvertToProxyLink(serverUrl, tracker.TrackerId, "blackhole"); - results.Add(item); } } diff --git a/src/Jackett/Services/IndexerManagerService.cs b/src/Jackett/Services/IndexerManagerService.cs index 276c24d3c..32657ee7d 100644 --- a/src/Jackett/Services/IndexerManagerService.cs +++ b/src/Jackett/Services/IndexerManagerService.cs @@ -87,6 +87,7 @@ namespace Jackett.Services var indexer = GetIndexer(name); var browseQuery = new TorznabQuery(); var results = await indexer.PerformQuery(browseQuery); + results = indexer.CleanLinks(results); logger.Info(string.Format("Found {0} releases from {1}", results.Count(), indexer.DisplayName)); if (results.Count() == 0) throw new Exception("Found no results while trying to browse this tracker"); diff --git a/src/Jackett/Services/ServerService.cs b/src/Jackett/Services/ServerService.cs index 1bf2dd6d4..c3a324e9c 100644 --- a/src/Jackett/Services/ServerService.cs +++ b/src/Jackett/Services/ServerService.cs @@ -20,6 +20,7 @@ using System.Net.NetworkInformation; using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; +using System.Web; namespace Jackett.Services { @@ -31,6 +32,7 @@ namespace Jackett.Services void ReserveUrls(bool doInstall = true); ServerConfig Config { get; } void SaveConfig(); + Uri ConvertToProxyLink(Uri link, string serverUrl, string indexerId, string action = "dl", string file = "/t.torrent"); } public class ServerService : IServerService @@ -63,6 +65,16 @@ namespace Jackett.Services get { return config; } } + public Uri ConvertToProxyLink(Uri link, string serverUrl, string indexerId, string action = "dl", string file = "/t.torrent") + { + if (link == null || (link.IsAbsoluteUri && link.Scheme == "magnet")) + return link; + + var encodedLink = HttpServerUtility.UrlTokenEncode(Encoding.UTF8.GetBytes(link.ToString())) + file; + var proxyLink = string.Format("{0}{1}/{2}/{3}/{4}", serverUrl, action, indexerId, config.APIKey, encodedLink); + return new Uri(proxyLink); + } + private void LoadConfig() { // Load config diff --git a/src/Jackett/Startup.cs b/src/Jackett/Startup.cs index 5a6419223..efa814c37 100644 --- a/src/Jackett/Startup.cs +++ b/src/Jackett/Startup.cs @@ -51,7 +51,7 @@ namespace Jackett // Configure Web API for self-host. var config = new HttpConfiguration(); - appBuilder.Use(); + appBuilder.Use(); // Setup tracing if enabled if (TracingEnabled) @@ -111,25 +111,27 @@ namespace Jackett config.Routes.MapHttpRoute( name: "download", - routeTemplate: "api/{indexerID}/download/{path}/t.torrent", + routeTemplate: "dl/{indexerID}/{apikey}/{path}/t.torrent", defaults: new { controller = "Download", action = "Download" } ); config.Routes.MapHttpRoute( name: "blackhole", - routeTemplate: "api/{indexerID}/blackhole/{path}/t.torrent", + routeTemplate: "bh/{indexerID}/{apikey}/{path}", defaults: new { controller = "Blackhole", action = "Blackhole" } ); + appBuilder.UseWebApi(config); + + appBuilder.UseFileServer(new FileServerOptions { RequestPath = new PathString(string.Empty), FileSystem = new PhysicalFileSystem(Engine.ConfigService.GetContentFolder()), EnableDirectoryBrowsing = false, - + }); - appBuilder.UseWebApi(config); } } }