Socks proxy support (#2058)

* socks proxy implementaion through SocksWebProxy package

* after merge fixes
This commit is contained in:
Andy Simons
2017-11-06 15:51:26 +05:00
committed by flightlevel
parent 7ce1c4acfb
commit 2b32fb358c
13 changed files with 218 additions and 100 deletions

View File

@@ -11,6 +11,7 @@ using Jackett.Utils;
using System.Net;
using System.Threading;
using Jacket.Common;
using Jackett.Models.Config;
namespace Jackett
{
@@ -26,7 +27,7 @@ namespace Jackett
public HttpMethod Method { get; private set; }
public IEnumerable<KeyValuePair<string, string>> PostData { get; set; }
public Dictionary<string, string> Headers { get; set; }
public string RawPOSTDdata { get; set;}
public string RawPOSTDdata { get; set; }
public CurlRequest(HttpMethod method, string url, string cookies = null, string referer = null, Dictionary<string, string> headers = null, string rawPOSTData = null)
{
@@ -55,31 +56,31 @@ namespace Jackett
}
}
public static async Task<CurlResponse> GetAsync(string url, string cookies = null, string referer = null, Dictionary<string, string> headers = null)
public static async Task<CurlResponse> GetAsync(string url, ServerConfig config, string cookies = null, string referer = null, Dictionary<string, string> headers = null)
{
var curlRequest = new CurlRequest(HttpMethod.Get, url, cookies, referer, headers);
var result = await PerformCurlAsync(curlRequest);
var result = await PerformCurlAsync(curlRequest, config);
return result;
}
public static async Task<CurlResponse> PostAsync(string url, IEnumerable<KeyValuePair<string, string>> formData, string cookies = null, string referer = null, Dictionary<string, string> headers = null, string rawPostData =null)
public static async Task<CurlResponse> PostAsync(string url, ServerConfig config, IEnumerable<KeyValuePair<string, string>> formData, string cookies = null, string referer = null, Dictionary<string, string> headers = null, string rawPostData = null)
{
var curlRequest = new CurlRequest(HttpMethod.Post, url, cookies, referer, headers);
curlRequest.PostData = formData;
curlRequest.RawPOSTDdata = rawPostData;
var result = await PerformCurlAsync(curlRequest);
var result = await PerformCurlAsync(curlRequest, config);
return result;
}
public static async Task<CurlResponse> PerformCurlAsync(CurlRequest curlRequest)
public static async Task<CurlResponse> PerformCurlAsync(CurlRequest curlRequest, ServerConfig config)
{
return await Task.Run(() => PerformCurl(curlRequest));
return await Task.Run(() => PerformCurl(curlRequest, config));
}
public delegate void ErrorMessage(string s);
public static ErrorMessage OnErrorMessage;
public static CurlResponse PerformCurl(CurlRequest curlRequest)
public static CurlResponse PerformCurl(CurlRequest curlRequest, ServerConfig config)
{
lock (instance)
{
@@ -88,13 +89,12 @@ namespace Jackett
using (var easy = new CurlEasy())
{
easy.Url = curlRequest.Url;
easy.BufferSize = 64 * 1024;
easy.UserAgent = BrowserUtil.ChromeUserAgent;
easy.FollowLocation = false;
easy.ConnectTimeout = 20;
if(curlRequest.Headers != null)
if (curlRequest.Headers != null)
{
CurlSlist curlHeaders = new CurlSlist();
foreach (var header in curlRequest.Headers)
@@ -154,10 +154,17 @@ namespace Jackett
easy.SetOpt(CurlOption.SslVerifyPeer, false);
}
if (JackettStartup.ProxyConnection != null)
var proxy = config.GetProxyUrl();
if (proxy != null)
{
easy.SetOpt(CurlOption.HttpProxyTunnel, 1);
easy.SetOpt(CurlOption.Proxy, JackettStartup.ProxyConnection);
easy.SetOpt(CurlOption.Proxy, proxy);
var authString = config.GetProxyAuthString();
if (authString != null)
{
easy.SetOpt(CurlOption.ProxyUserPwd, authString);
}
}
easy.Perform();
@@ -174,7 +181,7 @@ namespace Jackett
var headerBytes = Combine(headerBuffers.ToArray());
var headerString = Encoding.UTF8.GetString(headerBytes);
if (JackettStartup.ProxyConnection != null)
if (config.GetProxyUrl() != null)
{
var firstcrlf = headerString.IndexOf("\r\n\r\n");
var secondcrlf = headerString.IndexOf("\r\n\r\n", firstcrlf + 1);
@@ -210,7 +217,8 @@ namespace Jackett
if (key == "set-cookie")
{
var nameSplit = value.IndexOf('=');
if (nameSplit > -1) {
if (nameSplit > -1)
{
var cKey = value.Substring(0, nameSplit);
var cVal = value.Split(';')[0] + ";";
cookies.Add(new Tuple<string, string>(cKey, cVal));
@@ -242,12 +250,12 @@ namespace Jackett
OnErrorMessage("request.Cookies: " + curlRequest.Cookies);
OnErrorMessage("request.Referer: " + curlRequest.Referer);
OnErrorMessage("request.RawPOSTDdata: " + curlRequest.RawPOSTDdata);
OnErrorMessage("cookies: "+ cookieBuilder.ToString().Trim());
OnErrorMessage("cookies: " + cookieBuilder.ToString().Trim());
OnErrorMessage("headerString:\n" + headerString);
foreach (var headerPart in headerParts)
{
OnErrorMessage("headerParts: "+headerPart);
OnErrorMessage("headerParts: " + headerPart);
}
}
catch (Exception ex)
@@ -255,7 +263,7 @@ namespace Jackett
OnErrorMessage(string.Format("CurlHelper: error while handling NotImplemented/InternalServerError:\n{0}", ex));
}
}
var contentBytes = Combine(contentBuffers.ToArray());
var curlResponse = new CurlResponse(headers, contentBytes, status, cookieBuilder.ToString().Trim());
return curlResponse;

View File

@@ -18,6 +18,7 @@
<PackageReference Include="MimeMapping" Version="1.0.1.10" />
<PackageReference Include="Newtonsoft.Json" Version="10.0.3" />
<PackageReference Include="NLog" Version="5.0.0-beta11" />
<PackageReference Include="SocksWebProxy" Version="1.0.5" />
<PackageReference Include="YamlDotNet" Version="4.2.2" />
</ItemGroup>

View File

@@ -0,0 +1,13 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Jackett.Common.Models.Config
{
public enum ProxyType
{
Http,
Socks4,
Socks5,
}
}

View File

@@ -1,4 +1,5 @@
using System;
using Jackett.Common.Models.Config;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography;
@@ -27,30 +28,60 @@ namespace Jackett.Models.Config
public string OmdbApiKey { get; set; }
public string ProxyUrl { get; set; }
public ProxyType ProxyType { get; set; }
public int? ProxyPort { get; set; }
public string ProxyUsername { get; set; }
public string ProxyPassword { get; set; }
public string Proxy
public bool ProxyIsAnonymous
{
get
{
var proxy = ProxyUrl;
if (!string.IsNullOrWhiteSpace(ProxyUsername) && !string.IsNullOrWhiteSpace(ProxyPassword))
{
proxy = $"{ProxyUsername}:{ProxyPassword}@{proxy}";
}
if (ProxyPort.HasValue)
{
proxy = $"{proxy}:{ProxyPort}";
}
return proxy;
return string.IsNullOrWhiteSpace(ProxyUsername) || string.IsNullOrWhiteSpace(ProxyPassword);
}
}
public string GetProxyAuthString()
{
if (!ProxyIsAnonymous)
{
return $"{ProxyUsername}:{ProxyPassword}";
}
return null;
}
public string GetProxyUrl(bool withCreds = false)
{
var url = ProxyUrl;
if (string.IsNullOrWhiteSpace(url))
{
return null;
}
//remove protocol from url
var index = url.IndexOf("://");
if (index > -1)
{
url = url.Substring(index + 3);
}
url = ProxyPort.HasValue ? $"{url}:{ProxyPort}" : url;
var authString = GetProxyAuthString();
if (withCreds && authString != null)
{
url = $"{authString}@{url}";
}
if (ProxyType != ProxyType.Http)
{
var protocol = (Enum.GetName(typeof(ProxyType), ProxyType) ?? "").ToLower();
if (!string.IsNullOrEmpty(protocol))
{
url = $"{protocol}://{url}";
}
}
return url;
}
public string[] GetListenAddresses(bool? external = null)
{
if (external == null)

View File

@@ -1,6 +1,8 @@
using System.Collections.Generic;
using Jacket.Common;
using Jackett.Services;
using Jackett.Models.Config;
using Jackett.Common.Models.Config;
namespace Jackett.Models.DTO
{
@@ -19,6 +21,7 @@ namespace Jackett.Models.DTO
public string omdbkey { get; set; }
public string app_version { get; set; }
public ProxyType proxy_type { get; set; }
public string proxy_url { get; set; }
public int? proxy_port { get; set; }
public string proxy_username { get; set; }
@@ -44,6 +47,7 @@ namespace Jackett.Models.DTO
omdbkey = config.OmdbApiKey;
app_version = version;
proxy_type = config.ProxyType;
proxy_url = config.ProxyUrl;
proxy_port = config.ProxyPort;
proxy_username = config.ProxyUsername;

View File

@@ -1,8 +1,4 @@
using AutoMapper;
using CloudFlareUtilities;
using Jackett.Models;
using Jackett.Services;
using NLog;
using NLog;
using System;
using System.Collections.Generic;
using System.Linq;
@@ -11,11 +7,14 @@ using System.Net.Http;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Jackett.Services.Interfaces;
using Jacket.Common;
using Jackett.Models.Config;
using CloudFlareUtilities;
using com.LandonKey.SocksWebProxy;
using com.LandonKey.SocksWebProxy.Proxy;
using Jackett.Common.Models.Config;
namespace Jackett.Utils.Clients
{
@@ -85,31 +84,48 @@ namespace Jackett.Utils.Clients
}
}
var useProxy = false;
WebProxy proxyServer = null;
var proxyUrl = serverConfig.ProxyUrl;
if (!string.IsNullOrWhiteSpace(proxyUrl))
{
if (serverConfig.ProxyPort.HasValue)
{
proxyServer = new WebProxy(proxyUrl, serverConfig.ProxyPort.Value);
}
else
{
proxyServer = new WebProxy(proxyUrl);
}
var username = serverConfig.ProxyUsername;
var password = serverConfig.ProxyPassword;
if (!string.IsNullOrWhiteSpace(username) && !string.IsNullOrWhiteSpace(password))
{
var creds = new NetworkCredential(username, password);
proxyServer.Credentials = creds;
}
proxyServer.BypassProxyOnLocal = false;
useProxy = true;
}
using (ClearanceHandler clearanceHandlr = new ClearanceHandler())
{
IWebProxy proxyServer = null;
var proxyUrl = serverConfig.GetProxyUrl();
if (!string.IsNullOrWhiteSpace(proxyUrl))
{
useProxy = true;
NetworkCredential creds = null;
if (!serverConfig.ProxyIsAnonymous)
{
var username = serverConfig.ProxyUsername;
var password = serverConfig.ProxyPassword;
creds = new NetworkCredential(username, password);
}
if (serverConfig.ProxyType != ProxyType.Http)
{
var addresses = await Dns.GetHostAddressesAsync(serverConfig.ProxyUrl);
var socksConfig = new ProxyConfig
{
SocksAddress = addresses.FirstOrDefault(),
Username = serverConfig.ProxyUsername,
Password = serverConfig.ProxyPassword,
Version = serverConfig.ProxyType == ProxyType.Socks4 ?
ProxyConfig.SocksVersion.Four :
ProxyConfig.SocksVersion.Five
};
if (serverConfig.ProxyPort.HasValue)
{
socksConfig.SocksPort = serverConfig.ProxyPort.Value;
}
proxyServer = new SocksWebProxy(socksConfig, false);
}
else
{
proxyServer = new WebProxy(proxyUrl)
{
BypassProxyOnLocal = false,
Credentials = creds
};
}
}
using (HttpClientHandler clientHandlr = new HttpClientHandler
{
CookieContainer = cookies,
@@ -120,7 +136,6 @@ namespace Jackett.Utils.Clients
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
})
{
clearanceHandlr.InnerHandler = clientHandlr;
using (var client = new HttpClient(clearanceHandlr))
{
@@ -176,8 +191,10 @@ namespace Jackett.Utils.Clients
using (response = await client.SendAsync(request))
{
var result = new WebClientByteResult();
result.Content = await response.Content.ReadAsByteArrayAsync();
var result = new WebClientByteResult
{
Content = await response.Content.ReadAsByteArrayAsync()
};
foreach (var header in response.Headers)
{
@@ -187,7 +204,7 @@ namespace Jackett.Utils.Clients
// some cloudflare clients are using a refresh header
// Pull it out manually
if (response.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh"))
if (response.StatusCode == HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh"))
{
var refreshHeaders = response.Headers.GetValues("Refresh");
var redirval = "";

View File

@@ -17,6 +17,9 @@ using System.Threading.Tasks;
using Jackett.Services.Interfaces;
using Jacket.Common;
using Jackett.Models.Config;
using com.LandonKey.SocksWebProxy.Proxy;
using com.LandonKey.SocksWebProxy;
using Jackett.Common.Models.Config;
namespace Jackett.Utils.Clients
{
@@ -40,27 +43,44 @@ namespace Jackett.Utils.Clients
cookies = new CookieContainer();
var useProxy = false;
WebProxy proxyServer = null;
var proxyUrl = serverConfig.ProxyUrl;
IWebProxy proxyServer = null;
var proxyUrl = serverConfig.GetProxyUrl();
if (!string.IsNullOrWhiteSpace(proxyUrl))
{
if (serverConfig.ProxyPort.HasValue)
useProxy = true;
NetworkCredential creds = null;
if (!serverConfig.ProxyIsAnonymous)
{
proxyServer = new WebProxy(proxyUrl, serverConfig.ProxyPort.Value);
var username = serverConfig.ProxyUsername;
var password = serverConfig.ProxyPassword;
creds = new NetworkCredential(username, password);
}
if (serverConfig.ProxyType != ProxyType.Http)
{
var addresses = Dns.GetHostAddressesAsync(serverConfig.ProxyUrl).Result;
var socksConfig = new ProxyConfig
{
SocksAddress = addresses.FirstOrDefault(),
Username = serverConfig.ProxyUsername,
Password = serverConfig.ProxyPassword,
Version = serverConfig.ProxyType == ProxyType.Socks4 ?
ProxyConfig.SocksVersion.Four :
ProxyConfig.SocksVersion.Five
};
if (serverConfig.ProxyPort.HasValue)
{
socksConfig.SocksPort = serverConfig.ProxyPort.Value;
}
proxyServer = new SocksWebProxy(socksConfig, false);
}
else
{
proxyServer = new WebProxy(proxyUrl);
proxyServer = new WebProxy(proxyUrl)
{
BypassProxyOnLocal = false,
Credentials = creds
};
}
var username = serverConfig.ProxyUsername;
var password = serverConfig.ProxyPassword;
if (!string.IsNullOrWhiteSpace(username) && !string.IsNullOrWhiteSpace(password))
{
var creds = new NetworkCredential(username, password);
proxyServer.Credentials = creds;
}
proxyServer.BypassProxyOnLocal = false;
useProxy = true;
}
clearanceHandlr = new ClearanceHandler();
@@ -162,7 +182,7 @@ namespace Jackett.Utils.Clients
if (!string.IsNullOrEmpty(webRequest.RawBody))
{
var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast<KeyValuePair<string,string>?>().FirstOrDefault();
var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast<KeyValuePair<string, string>?>().FirstOrDefault();
if (type.HasValue)
{
var str = new StringContent(webRequest.RawBody);
@@ -253,7 +273,7 @@ namespace Jackett.Utils.Clients
var nameSplit = value.IndexOf('=');
if (nameSplit > -1)
{
responseCookies.Add(new Tuple<string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';')))+";"));
responseCookies.Add(new Tuple<string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';'))) + ";"));
}
}

View File

@@ -84,7 +84,7 @@ namespace Jackett.Utils.Clients
await Task.Delay(5000);
// request clearanceUri to get cf_clearance cookie
var response = await CurlHelper.GetAsync(clearanceUri, request.Cookies, request.Referer);
var response = await CurlHelper.GetAsync(clearanceUri, serverConfig, request.Cookies, request.Referer);
logger.Info(string.Format("UnixLibCurlWebClient: received CloudFlare clearance cookie: {0}", response.Cookies));
// add new cf_clearance cookies to the original request
@@ -104,7 +104,7 @@ namespace Jackett.Utils.Clients
Jackett.CurlHelper.CurlResponse response;
if (request.Type == RequestType.GET)
{
response = await CurlHelper.GetAsync(request.Url, request.Cookies, request.Referer, request.Headers);
response = await CurlHelper.GetAsync(request.Url, serverConfig, request.Cookies, request.Referer, request.Headers);
}
else
{
@@ -117,7 +117,7 @@ namespace Jackett.Utils.Clients
logger.Debug("UnixLibCurlWebClient: Posting " + StringUtil.PostDataFromDict(request.PostData));
}
response = await CurlHelper.PostAsync(request.Url, request.PostData, request.Cookies, request.Referer, request.Headers, request.RawBody);
response = await CurlHelper.PostAsync(request.Url, serverConfig, request.PostData, request.Cookies, request.Referer, request.Headers, request.RawBody);
}
var result = new WebClientByteResult()

View File

@@ -35,13 +35,14 @@ namespace Jackett.Utils.Clients
override protected async Task<WebClientByteResult> Run(WebRequest request)
{
var args = new StringBuilder();
if (serverConfig.Proxy != null)
var proxy = serverConfig.GetProxyUrl(true);
if (proxy != null)
{
args.AppendFormat("-x '" + serverConfig.Proxy + "' ");
args.AppendFormat("-x '" + proxy + "' ");
}
args.AppendFormat("--url \"{0}\" ", request.Url);
if (request.EmulateBrowser == true)
args.AppendFormat("-i -sS --user-agent \"{0}\" ", BrowserUtil.ChromeUserAgent);
else
@@ -61,7 +62,8 @@ namespace Jackett.Utils.Clients
{
var postString = StringUtil.PostDataFromDict(request.PostData);
args.AppendFormat("--data \"{0}\" ", request.RawBody.Replace("\"", "\\\""));
} else if (request.PostData != null && request.PostData.Count() > 0)
}
else if (request.PostData != null && request.PostData.Count() > 0)
{
var postString = StringUtil.PostDataFromDict(request.PostData);
args.AppendFormat("--data \"{0}\" ", postString);
@@ -85,7 +87,7 @@ namespace Jackett.Utils.Clients
string stdout = null;
await Task.Run(() =>
{
stdout = processService.StartProcessAndGetOutput(System.Environment.OSVersion.Platform == PlatformID.Unix ? "curl" : "curl.exe", args.ToString() , true);
stdout = processService.StartProcessAndGetOutput(System.Environment.OSVersion.Platform == PlatformID.Unix ? "curl" : "curl.exe", args.ToString(), true);
});
var outputData = File.ReadAllBytes(tempFile);
@@ -99,10 +101,10 @@ namespace Jackett.Utils.Clients
if (JackettStartup.ProxyConnection != null)
{
// the proxy provided headers too so we need to split headers again
var headSplit1 = stdout.IndexOf("\r\n\r\n",headSplit + 4);
var headSplit1 = stdout.IndexOf("\r\n\r\n", headSplit + 4);
if (headSplit1 > 0)
{
headers = stdout.Substring(headSplit + 4,headSplit1 - (headSplit + 4));
headers = stdout.Substring(headSplit + 4, headSplit1 - (headSplit + 4));
headSplit = headSplit1;
}
}

View File

@@ -67,6 +67,7 @@ function loadJackettSettings() {
$("#app-version").html(data.app_version);
$("#jackett-port").val(data.port);
$("#jackett-proxy-type").val(data.proxy_type);
$("#jackett-proxy-url").val(data.proxy_url);
$("#jackett-proxy-port").val(data.proxy_port);
$("#jackett-proxy-username").val(data.proxy_username);
@@ -1134,6 +1135,7 @@ function bindUIButtons() {
var jackett_omdb_key = $("#jackett-omdbkey").val();
var jackett_proxy_url = $("#jackett-proxy-url").val();
var jackett_proxy_type = $("#jackett-proxy-type").val();
var jackett_proxy_port = $("#jackett-proxy-port").val();
var jackett_proxy_username = $("#jackett-proxy-username").val();
var jackett_proxy_password = $("#jackett-proxy-password").val();
@@ -1147,6 +1149,7 @@ function bindUIButtons() {
logging: jackett_logging,
basepathoverride: jackett_basepathoverride,
omdbkey: jackett_omdb_key,
proxy_type: jackett_proxy_type,
proxy_url: jackett_proxy_url,
proxy_port: jackett_proxy_port,
proxy_username: jackett_proxy_username,

View File

@@ -123,6 +123,14 @@
<input id="jackett-savedir" class="form-control input-right" type="text" value="" placeholder="c:\torrents\">
</div>
<div class="input-area">
<span class="input-header">Proxy type: </span>
<select id="jackett-proxy-type" class="form-control input-right">
<option value="0">http</option>
<option value="1">socks4</option>
<option value="2">socks5</option>
</select>
</div>
<div class="input-area">
<span class="input-header">Proxy url: </span>
<input id="jackett-proxy-url" class="form-control input-right" type="text" value="" placeholder="Blank to disable">
@@ -458,14 +466,14 @@
<script id="jackett-search-results" type="text/x-handlebars-template">
<hr />
<p>Your search was done using:
{{#each Indexers}}{{Name}}
{{#if Error}}
<p>Your search was done using:
{{#each Indexers}}{{Name}}
{{#if Error}}
(<span title="{{Error}}"><b>Error</b></span>)
{{else}}
({{Results}})
{{/if}}
, {{/each}}
{{else}}
({{Results}})
{{/if}}
, {{/each}}
</p>
<table id="jackett-search-results-datatable" class="dataTable compact cell-border hover stripe">
<thead>

View File

@@ -109,7 +109,8 @@ namespace Jackett.Controllers.V20
indexerService.InitAggregateIndexer();
}
if (config.proxy_url != serverConfig.ProxyUrl ||
if (config.proxy_type != serverConfig.ProxyType ||
config.proxy_url != serverConfig.ProxyUrl ||
config.proxy_port != serverConfig.ProxyPort ||
config.proxy_username != serverConfig.ProxyUsername ||
config.proxy_password != serverConfig.ProxyPassword)
@@ -118,6 +119,7 @@ namespace Jackett.Controllers.V20
throw new Exception("The port you have selected is invalid, it must be below 65535.");
serverConfig.ProxyUrl = config.proxy_url;
serverConfig.ProxyType = config.proxy_type;
serverConfig.ProxyPort = config.proxy_port;
serverConfig.ProxyUsername = config.proxy_username;
serverConfig.ProxyPassword = config.proxy_password;

View File

@@ -0,0 +1,9 @@
namespace Jackett.Models.Config
{
public enum ProxyType
{
Http,
Socks4,
Socks5,
}
}