mirror of
https://github.com/Jackett/Jackett.git
synced 2025-09-17 17:34:09 +02:00
Feature/netcore preparation (#2072)
* Use platform detection that works on mono 4.6+ * Move to use package reference for restoring nuget packages. * DateTimeRoutines does not have Nuget packages that support .NET Standard (and therefore .NET Core). We will have to include them for now until we can get rid of this dependency. * Start spliting some interfaces into their own files - this will help by allowing us to split them out in the future into a seperate project so the actual implementations can stay within their respective architectures when required * Move out common libraries * Few more tidy up tasks to get things working with .NET Standard * Restructure the solution layout * Encoding work to reduce rework later on platforms without Windows codepages (or require compliance with RFC1345) * Move folder structure around to have more natural layout of the solutions * DI server configuration to get rid of "temporary" hack and dependency circle for serverservice * Make all encoding consistent to match the expected encoding casing for earlier versions of mono.
This commit is contained in:
Nathan Holland
committed by
flightlevel
flightlevel
parent
47a2ffa313
commit
571c52a0f2
277
src/Jackett.Common/CurlHelper.cs
Normal file
277
src/Jackett.Common/CurlHelper.cs
Normal file
@@ -0,0 +1,277 @@
|
||||
using CurlSharp;
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using System.Net.Http.Headers;
|
||||
using Jackett.Utils;
|
||||
using System.Net;
|
||||
using System.Threading;
|
||||
using Jacket.Common;
|
||||
|
||||
namespace Jackett
|
||||
{
|
||||
public class CurlHelper
|
||||
{
|
||||
private static readonly object instance = new object();
|
||||
|
||||
public class CurlRequest
|
||||
{
|
||||
public string Url { get; private set; }
|
||||
public string Cookies { get; private set; }
|
||||
public string Referer { get; private set; }
|
||||
public HttpMethod Method { get; private set; }
|
||||
public IEnumerable<KeyValuePair<string, string>> PostData { get; set; }
|
||||
public Dictionary<string, string> Headers { get; set; }
|
||||
public string RawPOSTDdata { get; set;}
|
||||
|
||||
public CurlRequest(HttpMethod method, string url, string cookies = null, string referer = null, Dictionary<string, string> headers = null, string rawPOSTData = null)
|
||||
{
|
||||
Method = method;
|
||||
Url = url.Replace(" ", "+"); // avoids bad request to cloudflare for urls containing a space followed by H (" H")
|
||||
Cookies = cookies;
|
||||
Referer = referer;
|
||||
Headers = headers;
|
||||
RawPOSTDdata = rawPOSTData;
|
||||
}
|
||||
}
|
||||
|
||||
public class CurlResponse
|
||||
{
|
||||
public List<string[]> HeaderList { get; private set; }
|
||||
public byte[] Content { get; private set; }
|
||||
public HttpStatusCode Status { get; private set; }
|
||||
public string Cookies { set; get; }
|
||||
|
||||
public CurlResponse(List<string[]> headers, byte[] content, HttpStatusCode s, string cookies)
|
||||
{
|
||||
HeaderList = headers;
|
||||
Content = content;
|
||||
Status = s;
|
||||
Cookies = cookies;
|
||||
}
|
||||
}
|
||||
|
||||
public static async Task<CurlResponse> GetAsync(string url, string cookies = null, string referer = null, Dictionary<string, string> headers = null)
|
||||
{
|
||||
var curlRequest = new CurlRequest(HttpMethod.Get, url, cookies, referer, headers);
|
||||
var result = await PerformCurlAsync(curlRequest);
|
||||
return result;
|
||||
}
|
||||
|
||||
public static async Task<CurlResponse> PostAsync(string url, IEnumerable<KeyValuePair<string, string>> formData, string cookies = null, string referer = null, Dictionary<string, string> headers = null, string rawPostData =null)
|
||||
{
|
||||
var curlRequest = new CurlRequest(HttpMethod.Post, url, cookies, referer, headers);
|
||||
curlRequest.PostData = formData;
|
||||
curlRequest.RawPOSTDdata = rawPostData;
|
||||
var result = await PerformCurlAsync(curlRequest);
|
||||
return result;
|
||||
}
|
||||
|
||||
public static async Task<CurlResponse> PerformCurlAsync(CurlRequest curlRequest)
|
||||
{
|
||||
return await Task.Run(() => PerformCurl(curlRequest));
|
||||
}
|
||||
|
||||
public delegate void ErrorMessage(string s);
|
||||
public static ErrorMessage OnErrorMessage;
|
||||
|
||||
public static CurlResponse PerformCurl(CurlRequest curlRequest)
|
||||
{
|
||||
lock (instance)
|
||||
{
|
||||
var headerBuffers = new List<byte[]>();
|
||||
var contentBuffers = new List<byte[]>();
|
||||
|
||||
using (var easy = new CurlEasy())
|
||||
{
|
||||
|
||||
easy.Url = curlRequest.Url;
|
||||
easy.BufferSize = 64 * 1024;
|
||||
easy.UserAgent = BrowserUtil.ChromeUserAgent;
|
||||
easy.FollowLocation = false;
|
||||
easy.ConnectTimeout = 20;
|
||||
if(curlRequest.Headers != null)
|
||||
{
|
||||
CurlSlist curlHeaders = new CurlSlist();
|
||||
foreach (var header in curlRequest.Headers)
|
||||
{
|
||||
curlHeaders.Append(header.Key + ": " + header.Value);
|
||||
}
|
||||
easy.SetOpt(CurlOption.HttpHeader, curlHeaders);
|
||||
}
|
||||
|
||||
easy.WriteFunction = (byte[] buf, int size, int nmemb, object data) =>
|
||||
{
|
||||
contentBuffers.Add(buf);
|
||||
return size * nmemb;
|
||||
};
|
||||
|
||||
easy.HeaderFunction = (byte[] buf, int size, int nmemb, object extraData) =>
|
||||
{
|
||||
headerBuffers.Add(buf);
|
||||
return size * nmemb;
|
||||
};
|
||||
|
||||
if (!string.IsNullOrEmpty(curlRequest.Cookies))
|
||||
easy.Cookie = curlRequest.Cookies;
|
||||
|
||||
if (!string.IsNullOrEmpty(curlRequest.Referer))
|
||||
easy.Referer = curlRequest.Referer;
|
||||
|
||||
if (curlRequest.Method == HttpMethod.Post)
|
||||
{
|
||||
if (!string.IsNullOrEmpty(curlRequest.RawPOSTDdata))
|
||||
{
|
||||
easy.Post = true;
|
||||
easy.PostFields = curlRequest.RawPOSTDdata;
|
||||
easy.PostFieldSize = Encoding.UTF8.GetByteCount(curlRequest.RawPOSTDdata);
|
||||
}
|
||||
else
|
||||
{
|
||||
easy.Post = true;
|
||||
var postString = StringUtil.PostDataFromDict(curlRequest.PostData);
|
||||
easy.PostFields = postString;
|
||||
easy.PostFieldSize = Encoding.UTF8.GetByteCount(postString);
|
||||
}
|
||||
}
|
||||
|
||||
if (JackettStartup.DoSSLFix == true)
|
||||
{
|
||||
// http://stackoverflow.com/questions/31107851/how-to-fix-curl-35-cannot-communicate-securely-with-peer-no-common-encryptio
|
||||
// https://git.fedorahosted.org/cgit/mod_nss.git/plain/docs/mod_nss.html
|
||||
easy.SslCipherList = SSLFix.CipherList;
|
||||
easy.FreshConnect = true;
|
||||
easy.ForbidReuse = true;
|
||||
}
|
||||
|
||||
if (JackettStartup.IgnoreSslErrors == true)
|
||||
{
|
||||
easy.SetOpt(CurlOption.SslVerifyhost, false);
|
||||
easy.SetOpt(CurlOption.SslVerifyPeer, false);
|
||||
}
|
||||
|
||||
if (JackettStartup.ProxyConnection != null)
|
||||
{
|
||||
easy.SetOpt(CurlOption.HttpProxyTunnel, 1);
|
||||
easy.SetOpt(CurlOption.Proxy, JackettStartup.ProxyConnection);
|
||||
}
|
||||
|
||||
easy.Perform();
|
||||
|
||||
if (easy.LastErrorCode != CurlCode.Ok)
|
||||
{
|
||||
var message = "Error " + easy.LastErrorCode.ToString() + " " + easy.LastErrorDescription + " " + easy.ErrorBuffer;
|
||||
if (null != OnErrorMessage)
|
||||
OnErrorMessage(message);
|
||||
else
|
||||
Console.WriteLine(message);
|
||||
}
|
||||
}
|
||||
|
||||
var headerBytes = Combine(headerBuffers.ToArray());
|
||||
var headerString = Encoding.UTF8.GetString(headerBytes);
|
||||
if (JackettStartup.ProxyConnection != null)
|
||||
{
|
||||
var firstcrlf = headerString.IndexOf("\r\n\r\n");
|
||||
var secondcrlf = headerString.IndexOf("\r\n\r\n", firstcrlf + 1);
|
||||
if (secondcrlf > 0)
|
||||
{
|
||||
headerString = headerString.Substring(firstcrlf + 4, secondcrlf - (firstcrlf));
|
||||
}
|
||||
}
|
||||
var headerParts = headerString.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
var headers = new List<string[]>();
|
||||
var headerCount = 0;
|
||||
HttpStatusCode status = HttpStatusCode.NotImplemented;
|
||||
var cookieBuilder = new StringBuilder();
|
||||
var cookies = new List<Tuple<string, string>>();
|
||||
foreach (var headerPart in headerParts)
|
||||
{
|
||||
if (headerCount == 0)
|
||||
{
|
||||
var split = headerPart.Split(' ');
|
||||
if (split.Length < 2)
|
||||
throw new Exception("HTTP Header missing");
|
||||
var responseCode = int.Parse(headerPart.Split(' ')[1]);
|
||||
status = (HttpStatusCode)responseCode;
|
||||
}
|
||||
else
|
||||
{
|
||||
var keyVal = headerPart.Split(new char[] { ':' }, 2);
|
||||
if (keyVal.Length > 1)
|
||||
{
|
||||
var key = keyVal[0].ToLower().Trim();
|
||||
var value = keyVal[1].Trim();
|
||||
|
||||
if (key == "set-cookie")
|
||||
{
|
||||
var nameSplit = value.IndexOf('=');
|
||||
if (nameSplit > -1) {
|
||||
var cKey = value.Substring(0, nameSplit);
|
||||
var cVal = value.Split(';')[0] + ";";
|
||||
cookies.Add(new Tuple<string, string>(cKey, cVal));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
headers.Add(new[] { key, value });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
headerCount++;
|
||||
}
|
||||
|
||||
foreach (var cookieGroup in cookies.GroupBy(c => c.Item1))
|
||||
{
|
||||
cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
|
||||
}
|
||||
|
||||
// add some debug output to track down the problem causing people getting InternalServerError results
|
||||
if (status == HttpStatusCode.NotImplemented || status == HttpStatusCode.InternalServerError)
|
||||
{
|
||||
try
|
||||
{
|
||||
OnErrorMessage("got NotImplemented/InternalServerError");
|
||||
OnErrorMessage("request.Method: " + curlRequest.Method);
|
||||
OnErrorMessage("request.Url: " + curlRequest.Url);
|
||||
OnErrorMessage("request.Cookies: " + curlRequest.Cookies);
|
||||
OnErrorMessage("request.Referer: " + curlRequest.Referer);
|
||||
OnErrorMessage("request.RawPOSTDdata: " + curlRequest.RawPOSTDdata);
|
||||
OnErrorMessage("cookies: "+ cookieBuilder.ToString().Trim());
|
||||
OnErrorMessage("headerString:\n" + headerString);
|
||||
|
||||
foreach (var headerPart in headerParts)
|
||||
{
|
||||
OnErrorMessage("headerParts: "+headerPart);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
OnErrorMessage(string.Format("CurlHelper: error while handling NotImplemented/InternalServerError:\n{0}", ex));
|
||||
}
|
||||
}
|
||||
|
||||
var contentBytes = Combine(contentBuffers.ToArray());
|
||||
var curlResponse = new CurlResponse(headers, contentBytes, status, cookieBuilder.ToString().Trim());
|
||||
return curlResponse;
|
||||
}
|
||||
}
|
||||
|
||||
public static byte[] Combine(params byte[][] arrays)
|
||||
{
|
||||
byte[] ret = new byte[arrays.Sum(x => x.Length)];
|
||||
int offset = 0;
|
||||
foreach (byte[] data in arrays)
|
||||
{
|
||||
Buffer.BlockCopy(data, 0, ret, offset, data.Length);
|
||||
offset += data.Length;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user