shpaq/Jackett-Jackett
1
0
Fork 0
mirror of https://github.com/Jackett/Jackett.git synced 2025-09-17 17:34:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

mejortorrent: add movies search (#3334)

Browse Source 
* feat(mejortorrent): add movie search

fixes #3310
This commit is contained in:
Ivan de la Beldad Fernandez
2018-07-04 17:10:21 +02:00
committed by kaso17
parent 2dc5edbb55
commit a81ef63075
1 changed files with 239 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

295
src/Jackett.Common/Indexers/MejorTorrent.cs
View File

@@ -30,7 +30,8 @@ namespace Jackett.Common.Indexers
link: WebUri.AbsoluteUri, link: WebUri.AbsoluteUri,
caps: new TorznabCapabilities(TorznabCatType.TV, caps: new TorznabCapabilities(TorznabCatType.TV,
TorznabCatType.TVSD, TorznabCatType.TVSD,
TorznabCatType.TVHD), TorznabCatType.TVHD,
TorznabCatType.Movies),
configService: configService, configService: configService,
client: wc, client: wc,
logger: l, logger: l,
@@ -73,6 +74,7 @@ namespace Jackett.Common.Indexers
query.SearchTerm = ""; query.SearchTerm = "";
} }
query.SearchTerm = query.SearchTerm.Replace("'", ""); query.SearchTerm = query.SearchTerm.Replace("'", "");
var requester = new MejorTorrentRequester(this); var requester = new MejorTorrentRequester(this);
var tvShowScraper = new TvShowScraper(); var tvShowScraper = new TvShowScraper();
var seasonScraper = new SeasonScraper(); var seasonScraper = new SeasonScraper();
@@ -81,17 +83,39 @@ namespace Jackett.Common.Indexers
var downloadGenerator = new DownloadGenerator(requester, downloadScraper); var downloadGenerator = new DownloadGenerator(requester, downloadScraper);
var tvShowPerformer = new TvShowPerformer(requester, tvShowScraper, seasonScraper, downloadGenerator); var tvShowPerformer = new TvShowPerformer(requester, tvShowScraper, seasonScraper, downloadGenerator);
var rssPerformer = new RssPerformer(requester, rssScraper, seasonScraper, downloadGenerator); var rssPerformer = new RssPerformer(requester, rssScraper, seasonScraper, downloadGenerator);
var movieSearchScraper = new MovieSearchScraper();
var movieInfoScraper = new MovieInfoScraper();
var movieDownloadScraper = new MovieDownloadScraper();
var moviePerformer = new MoviePerformer(requester, movieSearchScraper, movieInfoScraper, movieDownloadScraper);
var releases = new List<ReleaseInfo>();
if (string.IsNullOrEmpty(query.SanitizedSearchTerm)) if (string.IsNullOrEmpty(query.SanitizedSearchTerm))
{ {
var releases = await rssPerformer.PerformQuery(query); releases = (await rssPerformer.PerformQuery(query)).ToList();
var movie = releases.First();
movie.Category.Add(TorznabCatType.Movies.ID);
releases.ToList().Add(movie);
if (releases.Count() == 0) if (releases.Count() == 0)
{ {
releases = await AliveCheck(tvShowPerformer); releases = (await AliveCheck(tvShowPerformer)).ToList();
} }
return releases; return releases;
} }
return await tvShowPerformer.PerformQuery(query);
if (query.Categories.Contains(TorznabCatType.Movies.ID) || query.Categories.Count() == 0)
{
releases.AddRange(await moviePerformer.PerformQuery(query));
}
if (query.Categories.Contains(TorznabCatType.TV.ID) ||
query.Categories.Contains(TorznabCatType.TVSD.ID) ||
query.Categories.Contains(TorznabCatType.TVHD.ID) ||
query.Categories.Count() == 0)
{
releases.AddRange(await tvShowPerformer.PerformQuery(query));
}
return releases;
} }
private async Task<IEnumerable<ReleaseInfo>> AliveCheck(TvShowPerformer tvShowPerformer) private async Task<IEnumerable<ReleaseInfo>> AliveCheck(TvShowPerformer tvShowPerformer)
@@ -119,23 +143,23 @@ namespace Jackett.Common.Indexers
T Extract(IHtmlDocument html); T Extract(IHtmlDocument html);
} }
class RssScraper : IScraper<IEnumerable<KeyValuePair<MejorTorrentReleaseInfo, Uri>>> class RssScraper : IScraper<IEnumerable<KeyValuePair<MTReleaseInfo, Uri>>>
{ {
private readonly string LinkQuerySelector = "a[href*=\"/serie\"]"; private readonly string LinkQuerySelector = "a[href*=\"/serie\"]";
public IEnumerable<KeyValuePair<MejorTorrentReleaseInfo, Uri>> Extract(IHtmlDocument html) public IEnumerable<KeyValuePair<MTReleaseInfo, Uri>> Extract(IHtmlDocument html)
{ {
var episodes = GetNewEpisodesScratch(html); var episodes = GetNewEpisodesScratch(html);
var links = GetLinks(html); var links = GetLinks(html);
var results = new List<KeyValuePair<MejorTorrentReleaseInfo, Uri>>(); var results = new List<KeyValuePair<MTReleaseInfo, Uri>>();
for (var i = 0; i < episodes.Count(); i++) for (var i = 0; i < episodes.Count(); i++)
{ {
results.Add(new KeyValuePair<MejorTorrentReleaseInfo, Uri>(episodes.ElementAt(i), links.ElementAt(i))); results.Add(new KeyValuePair<MTReleaseInfo, Uri>(episodes.ElementAt(i), links.ElementAt(i)));
} }
return results; return results;
} }
private List<MejorTorrentReleaseInfo> GetNewEpisodesScratch(IHtmlDocument html) private List<MTReleaseInfo> GetNewEpisodesScratch(IHtmlDocument html)
{ {
var tvShowsElements = html.QuerySelectorAll(LinkQuerySelector); var tvShowsElements = html.QuerySelectorAll(LinkQuerySelector);
var seasonLinks = tvShowsElements.Select(e => e.Attributes["href"].Value); var seasonLinks = tvShowsElements.Select(e => e.Attributes["href"].Value);
@@ -144,10 +168,10 @@ namespace Jackett.Common.Indexers
var qualities = GetQualities(html); var qualities = GetQualities(html);
var seasonsFirstEpisodesAndLast = GetSeasonsFirstEpisodesAndLast(html); var seasonsFirstEpisodesAndLast = GetSeasonsFirstEpisodesAndLast(html);
var episodes = new List<MejorTorrentReleaseInfo>(); var episodes = new List<MTReleaseInfo>();
for(var i = 0; i < tvShowsElements.Count(); i++) for(var i = 0; i < tvShowsElements.Count(); i++)
{ {
var e = new MejorTorrentReleaseInfo(); var e = new MTReleaseInfo();
e.TitleOriginal = titles.ElementAt(i); e.TitleOriginal = titles.ElementAt(i);
e.PublishDate = dates.ElementAt(i); e.PublishDate = dates.ElementAt(i);
e.CategoryText = qualities.ElementAt(i); e.CategoryText = qualities.ElementAt(i);
@@ -300,9 +324,9 @@ namespace Jackett.Common.Indexers
} }
} }
class SeasonScraper : IScraper<IEnumerable<MejorTorrentReleaseInfo>> class SeasonScraper : IScraper<IEnumerable<MTReleaseInfo>>
{ {
public IEnumerable<MejorTorrentReleaseInfo> Extract(IHtmlDocument html) public IEnumerable<MTReleaseInfo> Extract(IHtmlDocument html)
{ {
var episodesLinksHtml = html.QuerySelectorAll("a[href*=\"/serie-episodio-descargar-torrent\"]"); var episodesLinksHtml = html.QuerySelectorAll("a[href*=\"/serie-episodio-descargar-torrent\"]");
var episodesTexts = episodesLinksHtml.Select(l => l.TextContent).ToList(); var episodesTexts = episodesLinksHtml.Select(l => l.TextContent).ToList();
@@ -314,7 +338,7 @@ namespace Jackett.Common.Indexers
.Select(stringParts => new int[]{ Int32.Parse(stringParts[0]), Int32.Parse(stringParts[1]), Int32.Parse(stringParts[2]) }) .Select(stringParts => new int[]{ Int32.Parse(stringParts[0]), Int32.Parse(stringParts[1]), Int32.Parse(stringParts[2]) })
.Select(intParts => new DateTime(intParts[0], intParts[1], intParts[2])); .Select(intParts => new DateTime(intParts[0], intParts[1], intParts[2]));
var episodes = episodesLinks.Select(e => new MejorTorrentReleaseInfo()).ToList(); var episodes = episodesLinks.Select(e => new MTReleaseInfo()).ToList();
for (var i = 0; i < episodes.Count(); i++) for (var i = 0; i < episodes.Count(); i++)
{ {
@@ -326,7 +350,7 @@ namespace Jackett.Common.Indexers
return episodes; return episodes;
} }
private void GuessEpisodes(MejorTorrentReleaseInfo release, string episodeText) private void GuessEpisodes(MTReleaseInfo release, string episodeText)
{ {
var seasonEpisodeRegex = new Regex(@"(\d{1,2}).*?(\d{1,2})", RegexOptions.IgnoreCase); var seasonEpisodeRegex = new Regex(@"(\d{1,2}).*?(\d{1,2})", RegexOptions.IgnoreCase);
var matchSeasonEpisode = seasonEpisodeRegex.Match(episodeText); var matchSeasonEpisode = seasonEpisodeRegex.Match(episodeText);
@@ -350,7 +374,7 @@ namespace Jackett.Common.Indexers
} }
} }
private void ExtractLinkInfo(MejorTorrentReleaseInfo release, String link) private void ExtractLinkInfo(MTReleaseInfo release, String link)
{ {
// LINK FORMAT: /serie-episodio-descargar-torrent-${ID}-${TITLE}-${SEASON_NUMBER}x${EPISODE_NUMBER}[range].html // LINK FORMAT: /serie-episodio-descargar-torrent-${ID}-${TITLE}-${SEASON_NUMBER}x${EPISODE_NUMBER}[range].html
var regex = new Regex(@"\/serie-episodio-descargar-torrent-(\d+)-(.*)-(\d{1,2}).*(\d{1,2}).*\.html", RegexOptions.IgnoreCase); var regex = new Regex(@"\/serie-episodio-descargar-torrent-(\d+)-(.*)-(\d{1,2}).*(\d{1,2}).*\.html", RegexOptions.IgnoreCase);
@@ -410,22 +434,23 @@ namespace Jackett.Common.Indexers
} }
} }
class MejorTorrentReleaseInfo : ReleaseInfo class MTReleaseInfo : ReleaseInfo
{ {
public string MejorTorrentID; public string MejorTorrentID;
public bool IsMovie;
public int _season; public int _season;
public int _episodeNumber; public int _episodeNumber;
private string _categoryText; private string _categoryText;
private string _originalTitle; private string _originalTitle;
public MejorTorrentReleaseInfo() public MTReleaseInfo()
{ {
this.Category = new List<int>(); this.Category = new List<int>();
this.Grabs = 5; this.Grabs = 5;
this.Files = 1; this.Files = 1;
this.PublishDate = new DateTime(); this.PublishDate = new DateTime();
this.Peers = 10; this.Peers = 1;
this.Seeders = 10; this.Seeders = 1;
this.Size = ReleaseInfo.BytesFromGB(1); this.Size = ReleaseInfo.BytesFromGB(1);
this._originalTitle = ""; this._originalTitle = "";
} }
@@ -438,28 +463,36 @@ namespace Jackett.Common.Indexers
get { return _categoryText; } get { return _categoryText; }
set set
{ {
switch (value) if (IsMovie)
{ {
case "SDTV": Category.Add(TorznabCatType.Movies.ID);
Category.Add(TorznabCatType.TVSD.ID); _categoryText = value;
_categoryText = "SDTV"; }
break; else
case "HDTV": {
Category.Add(TorznabCatType.TVSD.ID); switch (value)
_categoryText = "SDTV"; {
break; case "SDTV":
case "HDTV-720p": Category.Add(TorznabCatType.TVSD.ID);
Category.Add(TorznabCatType.TVHD.ID); _categoryText = "SDTV";
_categoryText = "HDTV-720p"; break;
break; case "HDTV":
case "HDTV-1080p": Category.Add(TorznabCatType.TVSD.ID);
Category.Add(TorznabCatType.TVHD.ID); _categoryText = "SDTV";
_categoryText = "HDTV-1080p"; break;
break; case "HDTV-720p":
default: Category.Add(TorznabCatType.TVHD.ID);
Category.Add(TorznabCatType.TV.ID); _categoryText = "HDTV-720p";
_categoryText = "HDTV-720p"; break;
break; case "HDTV-1080p":
Category.Add(TorznabCatType.TVHD.ID);
_categoryText = "HDTV-1080p";
break;
default:
Category.Add(TorznabCatType.TV.ID);
_categoryText = "HDTV-720p";
break;
}
} }
TitleOriginal = _originalTitle; TitleOriginal = _originalTitle;
} }
@@ -475,15 +508,165 @@ namespace Jackett.Common.Indexers
_originalTitle = value; _originalTitle = value;
if (_originalTitle != "") if (_originalTitle != "")
{ {
Title = _originalTitle.Replace(' ', '.'); Title = _originalTitle;
Title = char.ToUpper(Title[0]) + Title.Substring(1); Title = char.ToUpper(Title[0]) + Title.Substring(1);
} }
var seasonAndEpisode = "S" + Season.ToString("00") + "E" + EpisodeNumber.ToString("00"); var seasonAndEpisode = "";
if (Files > 1) if (!Category.Contains(TorznabCatType.Movies.ID))
{ {
seasonAndEpisode += "-" + FinalEpisodeNumber.ToString("00"); seasonAndEpisode = "S" + Season.ToString("00") + "E" + EpisodeNumber.ToString("00");
if (Files > 1)
{
seasonAndEpisode += "-" + FinalEpisodeNumber.ToString("00");
}
} }
Title = String.Join(".", new List<string>() { Title, seasonAndEpisode, CategoryText, "Spanish" }); Title = String.Join(".", new List<string>() { Title, seasonAndEpisode, CategoryText, "Spanish" }.Where(s => s != ""));
}
}
}
class MoviePerformer : IPerformer
{
private IRequester requester;
private IScraper<IEnumerable<Uri>> movieSearchScraper;
private IScraper<MTReleaseInfo> movieInfoScraper;
private IScraper<Uri> movieDownloadScraper;
public MoviePerformer(
IRequester requester,
IScraper<IEnumerable<Uri>> movieSearchScraper,
IScraper<MTReleaseInfo> movieInfoScraper,
IScraper<Uri> movieDownloadScraper)
{
this.requester = requester;
this.movieSearchScraper = movieSearchScraper;
this.movieInfoScraper = movieInfoScraper;
this.movieDownloadScraper = movieDownloadScraper;
}
public async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
{
var uriSearch = CreateSearchUri(query.SearchTerm);
var htmlSearch = await requester.MakeRequest(uriSearch);
var moviesInfoUris = movieSearchScraper.Extract(htmlSearch);
var infoHtmlTasks = moviesInfoUris.Select(async u => await requester.MakeRequest(u));
var infoHtmls = await Task.WhenAll(infoHtmlTasks);
var movies = infoHtmls.Select(h => movieInfoScraper.Extract(h));
var tasks = movies.Select(async m =>
{
var html = await requester.MakeRequest(m.Link);
return new KeyValuePair<MTReleaseInfo, IHtmlDocument>(m, html);
});
var moviesWithHtml = await Task.WhenAll(tasks.ToArray());
movies = moviesWithHtml.Select(movieWithHtml =>
{
var movie = movieWithHtml.Key;
var html = movieWithHtml.Value;
movie.Link = movieDownloadScraper.Extract(html);
movie.Guid = movieWithHtml.Key.Link;
return movie;
});
return movies;
}
}
class MovieSearchScraper : IScraper<IEnumerable<Uri>>
{
public IEnumerable<Uri> Extract(IHtmlDocument html)
{
return html.QuerySelectorAll("a[href*=\"/peli-\"]")
.Select(e => e.GetAttribute("href"))
.Select(relativeUri => new Uri(WebUri, relativeUri));
}
}
class MovieInfoScraper : IScraper<MTReleaseInfo>
{
public MTReleaseInfo Extract(IHtmlDocument html)
{
var release = new MTReleaseInfo();
release.IsMovie = true;
var selectors = html.QuerySelectorAll("b");
var titleSelector = html.QuerySelector("span>b");
try
{
var title = titleSelector.TextContent;
if (title.Contains("("))
{
title = title.Substring(0, title.IndexOf("(")).Trim();
}
release.TitleOriginal = title;
}
catch { }
try
{
var year = selectors.Where(s => s.TextContent.ToLower().Contains("año"))
.First().NextSibling.TextContent.Trim();
release.TitleOriginal += " (" + year + ")";
} catch { }
try
{
var dateStr = selectors.Where(s => s.TextContent.ToLower().Contains("fecha"))
.First().NextSibling.TextContent.Trim();
var date = Convert.ToDateTime(dateStr);
release.PublishDate = date;
} catch { }
try
{
var sizeStr = selectors.Where(s => s.TextContent.ToLower().Contains("tamaño"))
.First().NextSibling.TextContent.Trim();
Regex rgx = new Regex(@"[^0-9,.]");
long size;
if (sizeStr.ToLower().Trim().EndsWith("mb"))
{
size = ReleaseInfo.BytesFromMB(float.Parse(rgx.Replace(sizeStr, "")));
}
else
{
sizeStr = rgx.Replace(sizeStr, "").Replace(",", ".");
size = ReleaseInfo.BytesFromGB(float.Parse(rgx.Replace(sizeStr, "")));
}
release.Size = size;
} catch { }
try
{
var category = selectors.Where(s => s.TextContent.ToLower().Contains("formato"))
.First().NextSibling.TextContent.Trim();
release.CategoryText = category;
} catch { }
try
{
var title = titleSelector.TextContent;
if (title.Contains("(") && title.Contains(")") && title.Contains("4k"))
{
release.CategoryText = "2160p";
}
}
catch { }
try
{
var link = html.QuerySelector("a[href*=\"sec=descargas\"]").GetAttribute("href");
release.Link = new Uri(WebUri, link);
release.Guid = release.Link;
}
catch { }
return release;
}
}
class MovieDownloadScraper : IScraper<Uri>
{
public Uri Extract(IHtmlDocument html)
{
try
{
return new Uri(WebUri, html.QuerySelector("a[href*=\".torrent\"]").GetAttribute("href"));
}
catch
{
return null;
} }
} }
} }
@@ -553,14 +736,14 @@ namespace Jackett.Common.Indexers
class RssPerformer : IPerformer class RssPerformer : IPerformer
{ {
private IRequester requester; private IRequester requester;
private IScraper<IEnumerable<KeyValuePair<MejorTorrentReleaseInfo, Uri>>> rssScraper; private IScraper<IEnumerable<KeyValuePair<MTReleaseInfo, Uri>>> rssScraper;
private IScraper<IEnumerable<MejorTorrentReleaseInfo>> seasonScraper; private IScraper<IEnumerable<MTReleaseInfo>> seasonScraper;
private IDownloadGenerator downloadGenerator; private IDownloadGenerator downloadGenerator;
public RssPerformer( public RssPerformer(
IRequester requester, IRequester requester,
IScraper<IEnumerable<KeyValuePair<MejorTorrentReleaseInfo, Uri>>> rssScraper, IScraper<IEnumerable<KeyValuePair<MTReleaseInfo, Uri>>> rssScraper,
IScraper<IEnumerable<MejorTorrentReleaseInfo>> seasonScraper, IScraper<IEnumerable<MTReleaseInfo>> seasonScraper,
IDownloadGenerator downloadGenerator) IDownloadGenerator downloadGenerator)
{ {
this.requester = requester; this.requester = requester;
@@ -586,7 +769,7 @@ namespace Jackett.Common.Indexers
return episodes; return episodes;
} }
private async Task AddMejorTorrentIDs(MejorTorrentReleaseInfo episode, Uri seasonUri) private async Task AddMejorTorrentIDs(MTReleaseInfo episode, Uri seasonUri)
{ {
var html = await requester.MakeRequest(seasonUri); var html = await requester.MakeRequest(seasonUri);
var newEpisodes = seasonScraper.Extract(html); var newEpisodes = seasonScraper.Extract(html);
@@ -605,13 +788,13 @@ namespace Jackett.Common.Indexers
{ {
private IRequester requester; private IRequester requester;
private IScraper<IEnumerable<Season>> tvShowScraper; private IScraper<IEnumerable<Season>> tvShowScraper;
private IScraper<IEnumerable<MejorTorrentReleaseInfo>> seasonScraper; private IScraper<IEnumerable<MTReleaseInfo>> seasonScraper;
private IDownloadGenerator downloadGenerator; private IDownloadGenerator downloadGenerator;
public TvShowPerformer( public TvShowPerformer(
IRequester requester, IRequester requester,
IScraper<IEnumerable<Season>> tvShowScraper, IScraper<IEnumerable<Season>> tvShowScraper,
IScraper<IEnumerable<MejorTorrentReleaseInfo>> seasonScraper, IScraper<IEnumerable<MTReleaseInfo>> seasonScraper,
IDownloadGenerator downloadGenerator) IDownloadGenerator downloadGenerator)
{ {
this.requester = requester; this.requester = requester;
@@ -668,7 +851,7 @@ namespace Jackett.Common.Indexers
return seasons.ToList(); return seasons.ToList();
} }
private async Task<List<MejorTorrentReleaseInfo>> GetEpisodes(TorznabQuery query, IEnumerable<Season> seasons) private async Task<List<MTReleaseInfo>> GetEpisodes(TorznabQuery query, IEnumerable<Season> seasons)
{ {
var episodesHtmlTasks = new Dictionary<Season, Task<IHtmlDocument>>(); var episodesHtmlTasks = new Dictionary<Season, Task<IHtmlDocument>>();
seasons.ToList().ForEach(season => seasons.ToList().ForEach(season =>
@@ -698,7 +881,7 @@ namespace Jackett.Common.Indexers
interface IDownloadGenerator interface IDownloadGenerator
{ {
Task AddDownloadLinks(IEnumerable<MejorTorrentReleaseInfo> episodes); Task AddDownloadLinks(IEnumerable<MTReleaseInfo> episodes);
} }
class DownloadGenerator : IDownloadGenerator class DownloadGenerator : IDownloadGenerator
@@ -712,7 +895,7 @@ namespace Jackett.Common.Indexers
this.downloadScraper = downloadScraper; this.downloadScraper = downloadScraper;
} }
public async Task AddDownloadLinks(IEnumerable<MejorTorrentReleaseInfo> episodes) public async Task AddDownloadLinks(IEnumerable<MTReleaseInfo> episodes)
{ {
var downloadRequester = new MejorTorrentDownloadRequesterDecorator(requester); var downloadRequester = new MejorTorrentDownloadRequesterDecorator(requester);
var downloadHtml = await downloadRequester.MakeRequest(episodes.Select(e => e.MejorTorrentID)); var downloadHtml = await downloadRequester.MakeRequest(episodes.Select(e => e.MejorTorrentID));