shpaq/Jackett-Jackett
1
0
Fork 0
mirror of https://github.com/Jackett/Jackett.git synced 2025-09-17 17:34:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fix parse errors in Fuzer and Hebits (#7506) resolves #7502

Browse Source
This commit is contained in:
garfield69
2020-03-05 19:45:27 +13:00
committed by GitHub
parent ca19d2219e
commit ae303a5d4e
2 changed files with 96 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

181
src/Jackett.Common/Indexers/Fuzer.cs
View File

@@ -6,13 +6,13 @@ using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using AngleSharp.Dom;
using AngleSharp.Html.Parser;
using Jackett.Common.Helpers;
using Jackett.Common.Models;
using Jackett.Common.Models.IndexerConfig;
using Jackett.Common.Services.Interfaces;
using Jackett.Common.Utils;
using Jackett.Common.Utils.Clients;
using Newtonsoft.Json.Linq;
using NLog;
@@ -20,13 +20,13 @@ namespace Jackett.Common.Indexers
{
public class Fuzer : BaseWebIndexer
{
public override string[] LegacySiteLinks { get; protected set; } = new string[] {
"https://fuzer.me/",
public override string[] LegacySiteLinks { get; protected set; } =
{
"https://fuzer.me/"
};
private string SearchUrl => SiteLink + "browse.php";
private string LoginUrl => SiteLink + "login.php";
private const int MAXPAGES = 3;
private new ConfigurationDataRecaptchaLogin configData
{
@@ -36,13 +36,13 @@ namespace Jackett.Common.Indexers
public Fuzer(IIndexerConfigurationService configService, Utils.Clients.WebClient w, Logger l, IProtectionService ps)
: base(name: "Fuzer",
description: "Fuzer is a private torrent website with israeli torrents.",
link: "https://www.fuzer.me/",
configService: configService,
client: w,
logger: l,
p: ps,
configData: new ConfigurationDataRecaptchaLogin())
description: "Fuzer is a private torrent website with israeli torrents.",
link: "https://www.fuzer.me/",
configService: configService,
client: w,
logger: l,
p: ps,
configData: new ConfigurationDataRecaptchaLogin())
{
Encoding = Encoding.GetEncoding("windows-1255");
Language = "he-il";
@@ -130,18 +130,14 @@ namespace Jackett.Common.Indexers
public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
{
LoadValuesFromJson(configJson);
if (!string.IsNullOrWhiteSpace(configData.Captcha.Cookie))
{
CookieHeader = configData.Captcha.Cookie;
try
{
var results = await PerformQuery(new TorznabQuery());
if (results.Count() == 0)
{
if (!results.Any())
throw new Exception("Your cookie did not work");
}
IsConfigured = true;
SaveConfig();
return IndexerConfigurationStatus.Completed;
@@ -154,67 +150,48 @@ namespace Jackett.Common.Indexers
}
var loginPage = await RequestStringWithCookies(LoginUrl, string.Empty);
var pairs = new Dictionary<string, string> {
{ "vb_login_username", configData.Username.Value },
{ "vb_login_password", "" },
{ "securitytoken", "guest" },
{ "do","login"},
{ "vb_login_md5password", StringUtil.Hash(configData.Password.Value).ToLower()},
{ "vb_login_md5password_utf", StringUtil.Hash(configData.Password.Value).ToLower()},
{ "cookieuser", "1" }
};
var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, null, LoginUrl);
await ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("images/loading.gif"), () =>
var pairs = new Dictionary<string, string>
{
var errorMessage = "Couldn't login";
throw new ExceptionWithConfigData(errorMessage, configData);
});
{"vb_login_username", configData.Username.Value},
{"vb_login_password", ""},
{"securitytoken", "guest"},
{"do", "login"},
{"vb_login_md5password", StringUtil.Hash(configData.Password.Value).ToLower()},
{"vb_login_md5password_utf", StringUtil.Hash(configData.Password.Value).ToLower()},
{"cookieuser", "1"}
};
var result = await RequestLoginAndFollowRedirect(LoginUrl, pairs, loginPage.Cookies, true, null, LoginUrl);
await ConfigureIfOK(result.Cookies, result.Content?.Contains("images/loading.gif") == true,
() => throw new ExceptionWithConfigData("Couldn't login", configData));
Thread.Sleep(2);
return IndexerConfigurationStatus.RequiresTesting;
}
protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
{
var results = await performRegularQuery(query);
if (results.Count() == 0 && !query.IsImdbQuery)
{
return await performHebrewQuery(query);
}
var results = await PerformRegularQueryAsync(query);
if (!results.Any() && !query.IsImdbQuery)
return await PerformHebrewQueryAsync(query);
return results;
}
private async Task<IEnumerable<ReleaseInfo>> performHebrewQuery(TorznabQuery query)
private async Task<IEnumerable<ReleaseInfo>> PerformHebrewQueryAsync(TorznabQuery query)
{
var name = await getHebName(query.SearchTerm);
var name = await GetHebNameAsync(query.SearchTerm);
if (string.IsNullOrEmpty(name))
{
return new List<ReleaseInfo>();
}
else
{
return await performRegularQuery(query, name);
}
return await PerformRegularQueryAsync(query, name);
}
private async Task<IEnumerable<ReleaseInfo>> performRegularQuery(TorznabQuery query, string hebName = null)
private async Task<IEnumerable<ReleaseInfo>> PerformRegularQueryAsync(TorznabQuery query, string hebName = null)
{
var releases = new List<ReleaseInfo>();
var searchurls = new List<string>();
var searchUrl = SearchUrl;
var queryCollection = new NameValueCollection();
var searchString = query.GetQueryString();
if (query.IsImdbQuery)
searchString = query.ImdbID;
if (hebName != null)
{
searchString = hebName + " - עונה " + query.Season + " פרק " + query.Episode;
}
searchUrl += "?";
if (!string.IsNullOrWhiteSpace(searchString))
{
@@ -222,11 +199,7 @@ namespace Jackett.Common.Indexers
searchUrl += "&query=" + strEncoded + "&matchquery=any";
}
foreach (var cat in MapTorznabCapsToTrackers(query))
{
searchUrl += "&c[]=" + cat;
}
searchUrl = MapTorznabCapsToTrackers(query).Aggregate(searchUrl, (current, cat) => $"{current}&c[]={cat}");
var data = await RequestStringWithCookiesAndRetry(searchUrl);
try
{
@@ -236,63 +209,43 @@ namespace Jackett.Common.Indexers
foreach (var row in rows)
{
var release = new ReleaseInfo();
var main_title_link = row.QuerySelector("div.main_title > a");
release.Title = main_title_link.GetAttribute("longtitle");
if (release.Title.IsNullOrEmptyOrWhitespace())
release.Title = main_title_link.TextContent;
var mainTitleLink = row.QuerySelector("div.main_title > a");
release.Title = mainTitleLink.GetAttribute("longtitle");
if (string.IsNullOrWhiteSpace(release.Title))
release.Title = mainTitleLink.TextContent;
release.MinimumRatio = 1;
release.MinimumSeedTime = 172800; // 48 hours
if (ParseUtil.TryCoerceInt(row.QuerySelector("td:nth-child(7) > div").TextContent, out var seeders))
{
release.Seeders = seeders;
if (ParseUtil.TryCoerceInt(row.QuerySelector("td:nth-child(8) > div").TextContent, out var peers))
{
release.Peers = peers + release.Seeders;
}
}
release.Grabs = ParseUtil.CoerceLong(row.QuerySelector("td:nth-child(5)").TextContent.Replace(",", ""));
release.Seeders = ParseUtil.CoerceInt(row.QuerySelector("td:nth-child(6)").TextContent.Replace(",", ""));
release.Peers = ParseUtil.CoerceInt(row.QuerySelector("td:nth-child(7)").TextContent.Replace(",", "")) + release.Seeders;
release.Peers = ParseUtil.CoerceInt(row.QuerySelector("td:nth-child(7)").TextContent.Replace(",", "")) +
release.Seeders;
var fullSize = row.QuerySelector("td:nth-child(4)").TextContent;
release.Size = ReleaseInfo.GetBytes(fullSize);
release.Comments = new Uri(SiteLink + row.QuerySelector("a.threadlink[href]").GetAttribute("href"));
release.Link = new Uri(SiteLink + row.QuerySelector("a:has(div.dlimg)").GetAttribute("href"));
release.Guid = release.Comments;
try
{
release.BannerUrl = new Uri(row.QuerySelector("a[imgsrc]").GetAttribute("imgsrc"));
}
catch (Exception)
{
// do nothing, some releases have invalid banner URLs, ignore the banners in this case
}
//some releases have invalid banner URLs, ignore the banners in this case
if (Uri.TryCreate(row.QuerySelector("a[imgsrc]").GetAttribute("imgsrc"),
UriKind.Absolute, out var banner))
release.BannerUrl = banner;
var dateStringAll = row.QuerySelector("div.up_info2").ChildNodes.Last().ToString();
var dateParts = dateStringAll.Split(' ');
var dateString = dateParts[dateParts.Length - 2] + " " + dateParts[dateParts.Length - 1];
release.PublishDate = DateTime.ParseExact(dateString, "dd/MM/yy HH:mm", CultureInfo.InvariantCulture);
var categoryLink = row.QuerySelector("a[href^=\"/browse.php?cat=\"]").GetAttribute("href");
var catid = ParseUtil.GetArgumentFromQueryString(categoryLink, "cat");
release.Category = MapTrackerCatToNewznab(catid);
if (row.QuerySelector("a[href^=\"?freeleech=1\"]") != null)
release.DownloadVolumeFactor = 0;
else
release.DownloadVolumeFactor = 1;
release.UploadVolumeFactor = 1;
var sub_title = row.QuerySelector("div.sub_title");
var imdb_link = sub_title.QuerySelector("span.imdb-inline > a");
release.Imdb = ParseUtil.GetLongFromString(imdb_link.GetAttribute("href"));
foreach(var element in sub_title.QuerySelectorAll("span.imdb-inline"))
element.Remove();
release.Description = sub_title.TextContent;
var subTitle = row.QuerySelector("div.sub_title");
var imdbLink = subTitle.QuerySelector("span.imdb-inline > a");
if (imdbLink != null)
release.Imdb = ParseUtil.GetLongFromString(imdbLink.GetAttribute("href"));
release.Description = subTitle.FirstChild.TextContent;
releases.Add(release);
}
}
@@ -304,37 +257,35 @@ namespace Jackett.Common.Indexers
return releases;
}
private async Task<string> getHebName(string searchTerm)
private async Task<string> GetHebNameAsync(string searchTerm)
{
const string site = "http://thetvdb.com";
var url = site + "/index.php?searchseriesid=&tab=listseries&function=Search&";
url += "string=" + searchTerm; // eretz + nehedert
var results = await RequestStringWithCookies(url);
var queryString = new NameValueCollection
{
{"searchseriesid", ""},
{"tab", "listseries" },
{ "function", "Search"},
{"string", searchTerm }, // eretz + nehedert
};
var site = new UriBuilder
{
Scheme = "http",
Host = "thetvdb.com",
Path = "index.php",
Query = queryString.GetQueryString()
};
var results = await RequestStringWithCookies(site.ToString());
var parser = new HtmlParser();
var dom = parser.ParseDocument(results.Content);
var rowCount = 0;
var rows = dom.QuerySelectorAll("#listtable > tbody > tr");
foreach (var row in rows)
foreach (var row in rows.Skip(1))
{
if (rowCount < 1)
{
rowCount++;
continue;
}
var link = row.QuerySelector("td:nth-child(1) > a");
if (link.TextContent.Trim().ToLower() == searchTerm.Trim().ToLower())
if (string.Equals(link.TextContent.Trim(), searchTerm.Trim(), StringComparison.CurrentCultureIgnoreCase))
{
var address = link.GetAttribute("href");
if (string.IsNullOrEmpty(address))
{ continue; }
var realAddress = site + address.Replace("lid=7", "lid=24");
var realData = await RequestStringWithCookies(realAddress);
continue;
var realDom = parser.ParseDocument(results.Content);
return realDom.QuerySelector("#content:nth-child(1) > h1").TextContent;
}

95
src/Jackett.Common/Indexers/Hebits.cs
View File

@@ -1,10 +1,10 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using AngleSharp.Dom;
using AngleSharp.Html.Parser;
using Jackett.Common.Helpers;
using Jackett.Common.Models;
@@ -18,7 +18,6 @@ namespace Jackett.Common.Indexers
{
public class Hebits : BaseWebIndexer
{
private string LoginUrl => SiteLink + "login.php";
private string LoginPostUrl => SiteLink + "takeloginAjax.php";
private string SearchUrl => SiteLink + "browse.php?sort=4&type=desc";
@@ -30,26 +29,24 @@ namespace Jackett.Common.Indexers
public Hebits(IIndexerConfigurationService configService, Utils.Clients.WebClient wc, Logger l, IProtectionService ps)
: base(name: "Hebits",
description: "The Israeli Tracker",
link: "https://hebits.net/",
caps: TorznabUtil.CreateDefaultTorznabTVCaps(),
configService: configService,
client: wc,
logger: l,
p: ps,
downloadBase: "https://hebits.net/",
configData: new ConfigurationDataBasicLogin())
description: "The Israeli Tracker",
link: "https://hebits.net/",
caps: TorznabUtil.CreateDefaultTorznabTVCaps(),
configService: configService,
client: wc,
logger: l,
p: ps,
downloadBase: "https://hebits.net/",
configData: new ConfigurationDataBasicLogin())
{
Encoding = Encoding.GetEncoding("windows-1255");
Language = "he-il";
Type = "private";
AddCategoryMapping(19, TorznabCatType.MoviesSD);
AddCategoryMapping(25, TorznabCatType.MoviesOther); // Israeli Content
AddCategoryMapping(20, TorznabCatType.MoviesDVD);
AddCategoryMapping(36, TorznabCatType.MoviesBluRay);
AddCategoryMapping(27, TorznabCatType.MoviesHD);
AddCategoryMapping(7, TorznabCatType.TVSD); // Israeli SDTV
AddCategoryMapping(24, TorznabCatType.TVSD); // English SDTV
AddCategoryMapping(1, TorznabCatType.TVHD); // Israel HDTV
@@ -59,22 +56,24 @@ namespace Jackett.Common.Indexers
public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
{
LoadValuesFromJson(configJson);
var pairs = new Dictionary<string, string> {
{ "username", configData.Username.Value },
{ "password", configData.Password.Value }
var pairs = new Dictionary<string, string>
{
{"username", configData.Username.Value},
{"password", configData.Password.Value}
};
// Get inital cookies
CookieHeader = string.Empty;
var result = await RequestLoginAndFollowRedirect(LoginPostUrl, pairs, CookieHeader, true, null, SiteLink);
await ConfigureIfOK(result.Cookies, result.Content != null && result.Content.Contains("OK"), () =>
{
var parser = new HtmlParser();
var dom = parser.ParseDocument(result.Content);
var errorMessage = dom.TextContent.Trim();
errorMessage += " attempts left. Please check your credentials.";
throw new ExceptionWithConfigData(errorMessage, configData);
});
await ConfigureIfOK(
result.Cookies, result.Content?.Contains("OK") == true, () =>
{
var parser = new HtmlParser();
var dom = parser.ParseDocument(result.Content);
var errorMessage = dom.TextContent.Trim();
errorMessage += " attempts left. Please check your credentials.";
throw new ExceptionWithConfigData(errorMessage, configData);
});
return IndexerConfigurationStatus.RequiresTesting;
}
@@ -83,82 +82,48 @@ namespace Jackett.Common.Indexers
var releases = new List<ReleaseInfo>();
var searchString = query.GetQueryString();
var searchUrl = SearchUrl;
if (!string.IsNullOrWhiteSpace(searchString))
{
searchUrl += "&search=" + WebUtilityHelpers.UrlEncode(searchString, Encoding);
}
string.Format(SearchUrl, WebUtilityHelpers.UrlEncode(searchString, Encoding));
var cats = MapTorznabCapsToTrackers(query);
if (cats.Count > 0)
{
foreach (var cat in cats)
{
searchUrl += "&c" + cat + "=1";
}
}
searchUrl = cats.Aggregate(searchUrl, (url, cat) => $"{url}&c{cat}=1");
var response = await RequestStringWithCookies(searchUrl);
try
{
var parser = new HtmlParser();
var dom = parser.ParseDocument(response.Content);
var rows = dom.QuerySelectorAll(".browse > div > div");
foreach (var row in rows)
{
var release = new ReleaseInfo();
var debug = row.InnerHtml;
release.MinimumRatio = 1;
release.MinimumSeedTime = 172800; // 48 hours
var qTitle = row.QuerySelector(".bTitle");
var titleParts = qTitle.TextContent.Split('/');
if (titleParts.Length >= 2)
release.Title = titleParts[1].Trim();
else
release.Title = titleParts[0].Trim();
release.Title = titleParts.Length >= 2 ? titleParts[1].Trim() : titleParts[0].Trim();
var qDetailsLink = qTitle.QuerySelector("a[href^=\"details.php\"]");
release.Comments = new Uri(SiteLink + qDetailsLink.GetAttribute("href"));
release.Link = new Uri(SiteLink + row.QuerySelector("a[href^=\"download.php\"]").GetAttribute("href"));
release.Guid = release.Link;
var dateString = row.QuerySelector("div:last-child").TextContent.Trim();
var pattern = "\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}";
var match = Regex.Match(dateString, pattern);
var match = Regex.Match(dateString, @"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}");
if (match.Success)
{
release.PublishDate = DateTime.ParseExact(match.Value, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
}
var sizeStr = row.QuerySelector(".bSize").TextContent;
var sizeStr = row.QuerySelector(".bSize").TextContent.Trim();
release.Size = ReleaseInfo.GetBytes(sizeStr);
release.Seeders = ParseUtil.CoerceInt(row.QuerySelector(".bUping").TextContent.Trim());
release.Peers = release.Seeders + ParseUtil.CoerceInt(row.QuerySelector(".bDowning").TextContent.Trim());
var files = row.QuerySelector("div.bFiles").LastChild.ToString();
var files = row.QuerySelector("div.bFiles").ChildNodes.Last().TextContent.Trim();
release.Files = ParseUtil.CoerceInt(files);
var grabs = row.QuerySelector("div.bFinish").LastChild.ToString();
var grabs = row.QuerySelector("div.bFinish").ChildNodes.Last().TextContent.Trim();
release.Grabs = ParseUtil.CoerceInt(grabs);
if (row.QuerySelector("img[src=\"/pic/free.jpg\"]") != null)
release.DownloadVolumeFactor = 0;
else
release.DownloadVolumeFactor = 1;
release.DownloadVolumeFactor = row.QuerySelector("img[src=\"/pic/free.jpg\"]") != null ? 0 : 1;
if (row.QuerySelector("img[src=\"/pic/triple.jpg\"]") != null)
release.UploadVolumeFactor = 3;
else if (row.QuerySelector("img[src=\"/pic/double.jpg\"]") != null)
release.UploadVolumeFactor = 2;
else
release.UploadVolumeFactor = 1;
releases.Add(release);
}
}