shpaq/Jackett-Jackett
1
0
Fork 0
mirror of https://github.com/Jackett/Jackett.git synced 2025-09-17 17:34:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

Complete BTNv1 and Fix Cloudfare cookies

Browse Source
This commit is contained in:
KZ
2015-09-30 21:21:43 +01:00
parent 53e680ea87
commit 982300ec6e
11 changed files with 220 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
src/Jackett/CurlHelper.cs
View File

@@ -24,13 +24,15 @@ namespace Jackett
public string Referer { get; private set; }
public HttpMethod Method { get; private set; }
public IEnumerable<KeyValuePair<string, string>> PostData { get; set; }
public string RawPOSTDdata { get; set;}
public CurlRequest(HttpMethod method, string url, string cookies = null, string referer = null)
public CurlRequest(HttpMethod method, string url, string cookies = null, string referer = null, string rawPOSTData = null)
{
Method = method;
Url = url;
Cookies = cookies;
Referer = referer;
RawPOSTDdata = rawPOSTData;
}
}
@@ -57,10 +59,11 @@ namespace Jackett
return result;
}
public static async Task<CurlResponse> PostAsync(string url, IEnumerable<KeyValuePair<string, string>> formData, string cookies = null, string referer = null)
public static async Task<CurlResponse> PostAsync(string url, IEnumerable<KeyValuePair<string, string>> formData, string cookies = null, string referer = null, string rawPostData =null)
{
var curlRequest = new CurlRequest(HttpMethod.Post, url, cookies, referer);
curlRequest.PostData = formData;
curlRequest.RawPOSTDdata = rawPostData;
var result = await PerformCurlAsync(curlRequest);
return result;
}
@@ -108,10 +111,19 @@ namespace Jackett
if (curlRequest.Method == HttpMethod.Post)
{
easy.Post = true;
var postString = StringUtil.PostDataFromDict(curlRequest.PostData);
easy.PostFields = postString;
easy.PostFieldSize = Encoding.UTF8.GetByteCount(postString);
if (!string.IsNullOrEmpty(curlRequest.RawPOSTDdata))
{
easy.Post = true;
easy.PostFields = curlRequest.RawPOSTDdata;
easy.PostFieldSize = Encoding.UTF8.GetByteCount(curlRequest.RawPOSTDdata);
}
else
{
easy.Post = true;
var postString = StringUtil.PostDataFromDict(curlRequest.PostData);
easy.PostFields = postString;
easy.PostFieldSize = Encoding.UTF8.GetByteCount(postString);
}
}
if (Startup.DoSSLFix == true)
@@ -142,6 +154,7 @@ namespace Jackett
var headerCount = 0;
HttpStatusCode status = HttpStatusCode.InternalServerError;
var cookieBuilder = new StringBuilder();
var cookies = new List<Tuple<string, string>>();
foreach (var headerPart in headerParts)
{
if (headerCount == 0)
@@ -162,7 +175,10 @@ namespace Jackett
if (key == "set-cookie")
{
cookieBuilder.AppendFormat("{0} ", value.Substring(0, value.IndexOf(';') + 1));
var nameSplit = value.IndexOf('=');
if (nameSplit > -1) {
cookies.Add(new Tuple<string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') + 1)));
}
}
else
{
@@ -174,8 +190,13 @@ namespace Jackett
headerCount++;
}
foreach (var cookieGroup in cookies.GroupBy(c => c.Item1))
{
cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
}
var contentBytes = Combine(contentBuffers.ToArray());
var curlResponse = new CurlResponse(headers, contentBytes, status, cookieBuilder.ToString().TrimEnd());
var curlResponse = new CurlResponse(headers, contentBytes, status, cookieBuilder.ToString().Trim());
return curlResponse;
}
}

12
src/Jackett/Indexers/BaseIndexer.cs
View File

@@ -311,7 +311,7 @@ namespace Jackett.Indexers
return await webclient.GetBytes(request);
}
protected async Task<WebClientStringResult> PostDataWithCookies(string url, IEnumerable<KeyValuePair<string, string>> data, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null, string rawbody = null)
protected async Task<WebClientStringResult> PostDataWithCookies(string url, IEnumerable<KeyValuePair<string, string>> data, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null, string rawbody = null, bool? emulateBrowser = null)
{
var request = new Utils.Clients.WebRequest()
{
@@ -320,19 +320,23 @@ namespace Jackett.Indexers
Cookies = cookieOverride ?? CookieHeader,
PostData = data,
Referer = referer,
Headers = headers
Headers = headers,
RawBody = rawbody
};
if (emulateBrowser.HasValue)
request.EmulateBrowser = emulateBrowser.Value;
return await webclient.GetString(request);
}
protected async Task<WebClientStringResult> PostDataWithCookiesAndRetry(string url, IEnumerable<KeyValuePair<string, string>> data, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null, string rawbody = null)
protected async Task<WebClientStringResult> PostDataWithCookiesAndRetry(string url, IEnumerable<KeyValuePair<string, string>> data, string cookieOverride = null, string referer = null, Dictionary<string, string> headers = null, string rawbody = null, bool? emulateBrowser = null)
{
Exception lastException = null;
for (int i = 0; i < 3; i++)
{
try
{
return await PostDataWithCookies(url, data, cookieOverride, referer, headers, rawbody);
return await PostDataWithCookies(url, data, cookieOverride, referer, headers, rawbody, emulateBrowser);
}
catch (Exception e)
{

145
src/Jackett/Indexers/BroadcastTheNet.cs
View File

@@ -16,6 +16,7 @@ using System.Threading.Tasks;
using System.Web;
using Jackett.Models.IndexerConfig;
using System.Dynamic;
using Newtonsoft.Json;
namespace Jackett.Indexers
{
@@ -64,9 +65,9 @@ namespace Jackett.Indexers
}
private string JsonRPCRequest(string method, dynamic parameters)
private string JsonRPCRequest(string method, JArray parameters)
{
dynamic request = new ExpandoObject();
dynamic request = new JObject();
request["jsonrpc"] = "2.0";
request["method"] = method;
request["params"] = parameters;
@@ -76,76 +77,98 @@ namespace Jackett.Indexers
public async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
{
var searchString = query.GetQueryString();
var releases = new List<ReleaseInfo>();
var parameters = new JArray();
parameters.Add(new JValue(configData.Key.Value));
parameters.Add(new JValue(searchString.Trim()));
parameters.Add(new JValue(100));
parameters.Add(new JValue(0));
var response = await PostDataWithCookiesAndRetry(APIBASE, null, null, null, new Dictionary<string, string>()
{
{ "Accept", "application/json-rpc, application/json"},
{"ContentType", "application/json-rpc"}
}, JsonRPCRequest("getTorrents", new Object[]
{"Content-Type", "application/json-rpc"}
}, JsonRPCRequest("getTorrents", parameters),false);
try
{
configData.Key.Value
}));
var btnResponse = JsonConvert.DeserializeObject<BTNRPCResponse>(response.Content);
/*
var searchUrl = BrowsePage;
if (!string.IsNullOrWhiteSpace(query.GetQueryString()))
{
searchUrl += string.Format(QueryString, HttpUtility.UrlEncode(query.GetQueryString()));
}
var results = await RequestStringWithCookiesAndRetry(searchUrl);
try
{
CQ dom = results.Content;
var rows = dom["#sortabletable tr"];
foreach (var row in rows.Skip(1))
{
var release = new ReleaseInfo();
var qRow = row.Cq();
release.Title = qRow.Find(".tooltip-content div").First().Text();
if (string.IsNullOrWhiteSpace(release.Title))
continue;
release.Description = qRow.Find(".tooltip-content div").Get(1).InnerText.Trim();
var qLink = row.Cq().Find("td:eq(2) a:eq(1)");
release.Link = new Uri(qLink.Attr("href"));
release.Guid = release.Link;
release.Comments = new Uri(qRow.Find(".tooltip-target a").First().Attr("href"));
// 07-22-2015 11:08 AM
var dateString = qRow.Find("td:eq(1) div").Last().Children().Remove().End().Text().Trim();
release.PublishDate = DateTime.ParseExact(dateString, "MM-dd-yyyy hh:mm tt", CultureInfo.InvariantCulture);
var sizeStr = qRow.Find("td:eq(4)").Text().Trim();
release.Size = ReleaseInfo.GetBytes(sizeStr);
release.Seeders = ParseUtil.CoerceInt(qRow.Find("td:eq(6)").Text().Trim());
release.Peers = ParseUtil.CoerceInt(qRow.Find("td:eq(7)").Text().Trim()) + release.Seeders;
var catLink = row.Cq().Find("td:eq(0) a").First().Attr("href");
var catSplit = catLink.IndexOf("category=");
if (catSplit > -1)
{
catLink = catLink.Substring(catSplit + 9);
}
release.Category = MapTrackerCatToNewznab(catLink);
releases.Add(release);
}
}
catch (Exception ex)
{
OnParseError(results.Content, ex);
}*/
if (btnResponse != null && btnResponse.Result != null)
{
foreach (var itemKey in btnResponse.Result.Torrents)
{
var btnResult = itemKey.Value;
var item = new ReleaseInfo();
if (!string.IsNullOrEmpty(btnResult.SeriesBanner))
item.BannerUrl = new Uri(btnResult.SeriesBanner);
item.Category = TorznabCatType.TV.ID;
item.Comments = new Uri($"https://broadcasthe.net/torrents.php?id={btnResult.GroupID}&torrentid={btnResult.TorrentID}");
item.Description = btnResult.ReleaseName;
item.Guid = new Uri(btnResult.DownloadURL);
if (!string.IsNullOrWhiteSpace(btnResult.ImdbID))
item.Imdb = ParseUtil.CoerceLong(btnResult.ImdbID);
item.Link = new Uri(btnResult.DownloadURL);
item.MinimumRatio = 1;
item.PublishDate = DateTimeUtil.UnixTimestampToDateTime(btnResult.Time);
item.RageID = btnResult.TvrageID;
item.Seeders = btnResult.Seeders;
item.Peers = btnResult.Seeders + btnResult.Leechers;
item.Size = btnResult.Size;
item.TVDBId = btnResult.TvdbID;
item.Title = btnResult.ReleaseName;
releases.Add(item);
}
}
}
catch (Exception ex)
{
OnParseError(response.Content, ex);
}
return releases;
}
public class BTNRPCResponse
{
public string Id { get; set; }
public BTNResultPage Result { get; set; }
}
public class BTNResultPage
{
public Dictionary<int, BTNResultItem> Torrents { get; set; }
}
public class BTNResultItem
{
public int TorrentID { get; set; }
public string DownloadURL { get; set; }
public string GroupName { get; set; }
public int GroupID { get; set; }
public int SeriesID { get; set; }
public string Series { get; set; }
public string SeriesBanner { get; set; }
public string SeriesPoster { get; set; }
public string YoutubeTrailer { get; set; }
public string Category { get; set; }
public int? Snatched { get; set; }
public int? Seeders { get; set; }
public int? Leechers { get; set; }
public string Source { get; set; }
public string Container { get; set; }
public string Codec { get; set; }
public string Resolution { get; set; }
public string Origin { get; set; }
public string ReleaseName { get; set; }
public long Size { get; set; }
public long Time { get; set; }
public int? TvdbID { get; set; }
public int? TvrageID { get; set; }
public string ImdbID { get; set; }
public string InfoHash { get; set; }
}
}
}

2
src/Jackett/Indexers/NxtGn.cs
View File

@@ -193,7 +193,7 @@ namespace Jackett.Indexers
}
else if (imgUrl == "/pic/TV.png")
{
release.TheTvDbId = long.Parse(url.Substring(url.LastIndexOf('=') + 1));
release.TVDBId = long.Parse(url.Substring(url.LastIndexOf('=') + 1));
}
}
var nextPage = dom["#torrent-table-wrapper + p[align=center]"].Children().Last();

4
src/Jackett/Models/ReleaseInfo.cs
View File

@@ -21,11 +21,10 @@ namespace Jackett.Models
public long? Size { get; set; }
public string Description { get; set; }
public long? RageID { get; set; }
public long? TheTvDbId { get; set; }
public long? TVDBId { get; set; }
public long? Imdb { get; set; }
public int? Seeders { get; set; }
public int? Peers { get; set; }
public Uri ConverUrl { get; set; }
public Uri BannerUrl { get; set; }
public string InfoHash { get; set; }
public Uri MagnetUri { get; set; }
@@ -48,7 +47,6 @@ namespace Jackett.Models
Imdb = Imdb,
Seeders = Seeders,
Peers = Peers,
ConverUrl = ConverUrl,
BannerUrl = BannerUrl,
InfoHash = InfoHash,
MagnetUri = MagnetUri,

2
src/Jackett/Models/ResultPage.cs
View File

@@ -81,7 +81,7 @@ namespace Jackett.Models
),
getTorznabElement("magneturl", r.MagnetUri),
getTorznabElement("rageid", r.RageID),
getTorznabElement("thetvdb", r.TheTvDbId),
getTorznabElement("thetvdb", r.TVDBId),
getTorznabElement("imdb", r.Imdb),
getTorznabElement("seeders", r.Seeders),
getTorznabElement("peers", r.Peers),

4
src/Jackett/Utils/BrowserUtil.cs
View File

@@ -13,11 +13,11 @@ namespace Jackett.Utils
get {
if (System.Environment.OSVersion.Platform == PlatformID.Unix)
{
return "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chrome/44.0.2403.155 Safari/537.36";
return "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chrome/45.0.2454.101 Safari/537.36";
}
else
{
return "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36";
return "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36";
}
}
}

75
src/Jackett/Utils/Clients/HttpWebClient.cs
View File

@@ -1,5 +1,6 @@
using AutoMapper;
using Jackett.Models;
using Jackett.Services;
using NLog;
using System;
using System.Collections.Generic;
@@ -13,11 +14,13 @@ namespace Jackett.Utils.Clients
{
public class HttpWebClient : IWebClient
{
private Logger logger;
Logger logger;
IConfigurationService configService;
public HttpWebClient(Logger l)
public HttpWebClient(Logger l, IConfigurationService c)
{
logger = l;
configService = c;
}
@@ -41,13 +44,13 @@ namespace Jackett.Utils.Clients
return Mapper.Map<WebClientStringResult>(result);
}
private async Task<WebClientByteResult> Run(WebRequest request)
private async Task<WebClientByteResult> Run(WebRequest webRequest)
{
var cookies = new CookieContainer();
if (!string.IsNullOrEmpty(request.Cookies))
if (!string.IsNullOrEmpty(webRequest.Cookies))
{
var uri = new Uri(request.Url);
foreach (var c in request.Cookies.Split(';'))
var uri = new Uri(webRequest.Url);
foreach (var c in webRequest.Cookies.Split(';'))
{
try
{
@@ -67,27 +70,52 @@ namespace Jackett.Utils.Clients
UseCookies = true,
});
client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent);
if(webRequest.EmulateBrowser)
client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent);
else
client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion());
HttpResponseMessage response = null;
var request = new HttpRequestMessage();
request.Headers.ExpectContinue = false;
request.RequestUri = new Uri(webRequest.Url);
if (request.Headers != null)
if (webRequest.Headers != null)
{
foreach (var header in request.Headers)
foreach (var header in webRequest.Headers)
{
client.DefaultRequestHeaders.Add(header.Key, header.Value);
if (header.Key != "Content-Type")
{
request.Headers.Add(header.Key, header.Value);
}
}
}
if (request.Type == RequestType.POST)
if (!string.IsNullOrEmpty(webRequest.RawBody))
{
var content = new FormUrlEncodedContent(request.PostData);
response = await client.PostAsync(request.Url, content);
var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast<KeyValuePair<string,string>?>().FirstOrDefault();
if (type.HasValue)
{
var str = new StringContent(webRequest.RawBody);
str.Headers.Remove("Content-Type");
str.Headers.Add("Content-Type", type.Value.Value);
request.Content = str;
}
else
request.Content = new StringContent(webRequest.RawBody);
request.Method = HttpMethod.Post;
}
else if (webRequest.Type == RequestType.POST)
{
request.Content = new FormUrlEncodedContent(webRequest.PostData);
request.Method = HttpMethod.Post;
}
else
{
response = await client.GetAsync(request.Url);
request.Method = HttpMethod.Get;
}
response = await client.SendAsync(request);
var result = new WebClientByteResult();
result.Content = await response.Content.ReadAsByteArrayAsync();
if (response.Headers.Location != null)
@@ -100,18 +128,27 @@ namespace Jackett.Utils.Clients
// Pull it out manually ignoring the expiry date then set it manually
// http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
IEnumerable<string> cookieHeaders;
var responseCookies = new List<Tuple<string, string>>();
if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
{
var cookieBuilder = new StringBuilder();
foreach (var c in cookieHeaders)
foreach (var value in cookieHeaders)
{
cookieBuilder.AppendFormat("{0} ", c.Substring(0, c.IndexOf(';') + 1));
var nameSplit = value.IndexOf('=');
if (nameSplit > -1)
{
responseCookies.Add(new Tuple<string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') + 1)));
}
}
result.Cookies = cookieBuilder.ToString().TrimEnd();
var cookieBuilder = new StringBuilder();
foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1))
{
cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
}
result.Cookies = cookieBuilder.ToString().Trim();
}
ServerUtil.ResureRedirectIsFullyQualified(request, result);
ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
return result;
}
}

8
src/Jackett/Utils/Clients/UnixLibCurlWebClient.cs
View File

@@ -76,12 +76,16 @@ namespace Jackett.Utils.Clients
}
else
{
if (request.PostData != null && request.PostData.Count() > 0)
if (!string.IsNullOrEmpty(request.RawBody))
{
logger.Debug("UnixLibCurlWebClient: Posting " + request.RawBody);
}
else if (request.PostData != null && request.PostData.Count() > 0)
{
logger.Debug("UnixLibCurlWebClient: Posting " + StringUtil.PostDataFromDict(request.PostData));
}
response = await CurlHelper.PostAsync(request.Url, request.PostData, request.Cookies, request.Referer);
response = await CurlHelper.PostAsync(request.Url, request.PostData, request.Cookies, request.Referer, request.RawBody);
}
var result = new WebClientByteResult()

37
src/Jackett/Utils/Clients/UnixSafeCurlWebClient.cs
View File

@@ -16,13 +16,15 @@ namespace Jackett.Utils.Clients
{
public class UnixSafeCurlWebClient : IWebClient
{
private IProcessService processService;
private Logger logger;
IProcessService processService;
Logger logger;
IConfigurationService configService;
public UnixSafeCurlWebClient(IProcessService p, Logger l)
public UnixSafeCurlWebClient(IProcessService p, Logger l, IConfigurationService c)
{
processService = p;
logger = l;
configService = c;
}
public void Init()
@@ -49,7 +51,11 @@ namespace Jackett.Utils.Clients
{
var args = new StringBuilder();
args.AppendFormat("--url \"{0}\" ", request.Url);
args.AppendFormat("-i -sS --user-agent \"{0}\" ", BrowserUtil.ChromeUserAgent);
if (request.EmulateBrowser)
args.AppendFormat("-i -sS --user-agent \"{0}\" ", BrowserUtil.ChromeUserAgent);
else
args.AppendFormat("-i -sS --user-agent \"{0}\" ", "Jackett/" + configService.GetVersion());
if (!string.IsNullOrWhiteSpace(request.Cookies))
{
@@ -61,7 +67,11 @@ namespace Jackett.Utils.Clients
args.AppendFormat("--referer \"{0}\" ", request.Referer);
}
if (request.PostData != null && request.PostData.Count() > 0)
if (!string.IsNullOrEmpty(request.RawBody))
{
var postString = StringUtil.PostDataFromDict(request.PostData);
args.AppendFormat("--data \"{0}\" ", request.RawBody.Replace("\"", "\\\""));
} else if (request.PostData != null && request.PostData.Count() > 0)
{
var postString = StringUtil.PostDataFromDict(request.PostData);
args.AppendFormat("--data \"{0}\" ", postString);
@@ -92,6 +102,9 @@ namespace Jackett.Utils.Clients
throw new Exception("Invalid response");
var headers = stdout.Substring(0, headSplit);
var headerCount = 0;
var cookieBuilder = new StringBuilder();
var cookies = new List<Tuple<string, string>>();
foreach (var header in headers.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries))
{
if (headerCount == 0)
@@ -109,11 +122,11 @@ namespace Jackett.Utils.Clients
switch (name)
{
case "set-cookie":
var cookieDataSplit = value.IndexOf(';');
if (cookieDataSplit > 0)
var nameSplit = value.IndexOf('=');
if (nameSplit > -1)
{
result.Cookies += value.Substring(0, cookieDataSplit + 1) + " ";
}//Location
cookies.Add(new Tuple<string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') + 1)));
}
break;
case "location":
result.RedirectingTo = value.Trim();
@@ -124,6 +137,12 @@ namespace Jackett.Utils.Clients
headerCount++;
}
foreach (var cookieGroup in cookies.GroupBy(c => c.Item1))
{
cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
}
result.Cookies = cookieBuilder.ToString().Trim();
result.Content = new byte[outputData.Length - (headSplit + 3)];
var dest = 0;
for (int i = headSplit + 4; i < outputData.Length; i++)

4
src/Jackett/Utils/Clients/WebRequest.cs
View File

@@ -14,6 +14,7 @@ namespace Jackett.Utils.Clients
PostData = new List<KeyValuePair<string, string>>();
Type = RequestType.GET;
Headers = new Dictionary<string, string>();
EmulateBrowser = true;
}
public WebRequest(string url)
@@ -22,6 +23,7 @@ namespace Jackett.Utils.Clients
Type = RequestType.GET;
Url = url;
Headers = new Dictionary<string, string>();
EmulateBrowser = true;
}
public string Url { get; set; }
@@ -29,6 +31,8 @@ namespace Jackett.Utils.Clients
public string Cookies { get; set; }
public string Referer { get; set; }
public RequestType Type { get; set; }
public string RawBody { get; set; }
public bool EmulateBrowser { get; set; }
/// <summary>
/// Warning this is only implemented on HTTPWebClient currently!