core: FlareSolverr integration. Bypass Cloudflare protection (#9028)

src/Jackett.Common/Content/custom.js

@@ -106,6 +106,7 @@ function loadJackettSettings() {
             $("#jackett-show-releases").attr("disabled", true);
         }
 
+        $("#jackett-flaresolverrurl").val(data.flaresolverrurl);
         $("#jackett-omdbkey").val(data.omdbkey);
         $("#jackett-omdburl").val(data.omdburl);
         var password = data.password;
@@ -1137,6 +1138,7 @@ function bindUIButtons() {
         var jackett_cache_enabled = $("#jackett-cache-enabled").is(':checked');
         var jackett_cache_ttl = $("#jackett-cache-ttl").val();
         var jackett_cache_max_results_per_indexer = $("#jackett-cache-max-results-per-indexer").val();
+        var jackett_flaresolverr_url = $("#jackett-flaresolverrurl").val();
         var jackett_omdb_key = $("#jackett-omdbkey").val();
         var jackett_omdb_url = $("#jackett-omdburl").val();
 
@@ -1158,6 +1160,7 @@ function bindUIButtons() {
             cache_enabled: jackett_cache_enabled,
             cache_ttl: jackett_cache_ttl,
             cache_max_results_per_indexer: jackett_cache_max_results_per_indexer,
+            flaresolverrurl: jackett_flaresolverr_url,
             omdbkey: jackett_omdb_key,
             omdburl: jackett_omdb_url,
             proxy_type: jackett_proxy_type,

src/Jackett.Common/Content/index.html

@@ -200,6 +200,10 @@
             <span class="input-header">Cache max results per indexer: </span>
             <input id="jackett-cache-max-results-per-indexer" class="form-control input-right" type="text" value="" placeholder="">
         </div>
+        <div class="input-area">
+          <span class="input-header">FlareSolverr API URL: </span>
+          <input id="jackett-flaresolverrurl" class="form-control input-right" type="text" value="" placeholder="Blank for default">
+        </div>
         <div class="input-area">
             <span class="input-header">OMDB API key: </span>
             <input id="jackett-omdbkey" class="form-control input-right" type="text" value="" placeholder="">
@@ -692,6 +696,6 @@
     </script>
 
     <script type="text/javascript" src="../libs/api.js?changed=2017083001"></script>
-    <script type="text/javascript" src="../custom.js?changed=20201208"></script>
+    <script type="text/javascript" src="../custom.js?changed=20201220"></script>
 </body>
 </html>

src/Jackett.Common/Jackett.Common.csproj

@@ -14,7 +14,7 @@
     <PackageReference Include="Autofac" Version="5.2.0" />
     <PackageReference Include="AutoMapper" Version="8.1.1" />
     <PackageReference Include="BencodeNET" Version="3.1.4" />
-    <PackageReference Include="CloudflareSolverReNgosang" Version="1.0.7.3" />
+    <PackageReference Include="FlareSolverrSharp" Version="1.2.0" />
     <PackageReference Include="CommandLineParser" Version="2.8.0" />
     <PackageReference Include="DotNet4.SocksProxy" Version="1.4.0.1" />
     <PackageReference Include="Microsoft.AspNetCore.Http" Version="2.2.2" />

src/Jackett.Common/Models/Config/ServerConfig.cs

@@ -42,6 +42,7 @@ namespace Jackett.Common.Models.Config
         public bool CacheEnabled { get; set; }
         public long CacheTtl { get; set; }
         public long CacheMaxResultsPerIndexer { get; set; }
+        public string FlareSolverrUrl { get; set; }
         public string OmdbApiKey { get; set; }
         public string OmdbApiUrl { get; set; }
 

src/Jackett.Common/Models/DTO/ServerConfig.cs

@@ -34,6 +34,8 @@ namespace Jackett.Common.Models.DTO
         [DataMember]
         public long cache_max_results_per_indexer { get; set; }
         [DataMember]
+        public string flaresolverrurl { get; set; }
+        [DataMember]
         public string omdbkey { get; set; }
         [DataMember]
         public string omdburl { get; set; }
@@ -70,6 +72,7 @@ namespace Jackett.Common.Models.DTO
             cache_enabled = config.CacheEnabled;
             cache_ttl = config.CacheTtl;
             cache_max_results_per_indexer = config.CacheMaxResultsPerIndexer;
+            flaresolverrurl = config.FlareSolverrUrl;
             omdbkey = config.OmdbApiKey;
             omdburl = config.OmdbApiUrl;
             app_version = version;

src/Jackett.Common/Services/CacheService.cs

@@ -8,7 +8,6 @@ using Jackett.Common.Indexers;
 using Jackett.Common.Models;
 using Jackett.Common.Models.Config;
 using Jackett.Common.Services.Interfaces;
-using Jint.Parser;
 using NLog;
 
 namespace Jackett.Common.Services
@@ -218,7 +217,9 @@ namespace Jackett.Common.Services
                 var total = resultsPerQuery.Select(q => q.Item2).Sum();
                 if (total <= _serverConfig.CacheMaxResultsPerIndexer)
                     break;
-                trackerCache.Queries.Remove(resultsPerQuery.Pop().Item1); // remove the older
+                var olderQuery = resultsPerQuery.Last();
+                trackerCache.Queries.Remove(olderQuery.Item1); // remove the older
+                resultsPerQuery.Remove(olderQuery);
                 prunedCounter++;
             }
 

src/Jackett.Common/Utils/BrowserUtil.cs

@@ -7,8 +7,8 @@ namespace Jackett.Common.Utils
         // When updating these make sure they are not detected by the incapsula bot detection engine
         // (e.g. kickasstorrent indexer)
         public static string ChromeUserAgent => Environment.OSVersion.Platform == PlatformID.Unix ?
-            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36" :
+            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36" :
-            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36";
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36";
 
     }
 }

src/Jackett.Common/Utils/Clients/HttpWebClient.cs

@@ -8,7 +8,7 @@ using System.Net.Security;
 using System.Security.Cryptography.X509Certificates;
 using System.Text;
 using System.Threading.Tasks;
-using CloudflareSolverRe;
+using FlareSolverrSharp;
 using Jackett.Common.Helpers;
 using Jackett.Common.Models.Config;
 using Jackett.Common.Services.Interfaces;
@@ -77,9 +77,10 @@ namespace Jackett.Common.Utils.Clients
 
             var userAgent = webRequest.EmulateBrowser.Value ? BrowserUtil.ChromeUserAgent : "Jackett/" + configService.GetVersion();
 
-            using (var clearanceHandlr = new ClearanceHandler(userAgent))
+            using (var clearanceHandlr = new ClearanceHandler(serverConfig.FlareSolverrUrl))
             {
-                clearanceHandlr.MaxTries = 10;
+                clearanceHandlr.UserAgent = userAgent;
+                clearanceHandlr.MaxTimeout = 50000;
                 using (var clientHandlr = new HttpClientHandler
                 {
                     CookieContainer = cookies,

src/Jackett.Common/Utils/Clients/HttpWebClient2.cs

@@ -8,7 +8,7 @@ using System.Net.Security;
 using System.Security.Cryptography.X509Certificates;
 using System.Text;
 using System.Threading.Tasks;
-using CloudflareSolverRe;
+using FlareSolverrSharp;
 using Jackett.Common.Helpers;
 using Jackett.Common.Models.Config;
 using Jackett.Common.Services.Interfaces;
@@ -54,9 +54,10 @@ namespace Jackett.Common.Utils.Clients
 
         public void CreateClient()
         {
-            clearanceHandlr = new ClearanceHandler(BrowserUtil.ChromeUserAgent)
+            clearanceHandlr = new ClearanceHandler(serverConfig.FlareSolverrUrl)
             {
-                MaxTries = 10
+                UserAgent = BrowserUtil.ChromeUserAgent,
+                MaxTimeout = 50000
             };
             clientHandlr = new HttpClientHandler
             {

src/Jackett.Server/Controllers/ServerConfigurationController.cs

@@ -113,6 +113,13 @@ namespace Jackett.Server.Controllers
             serverConfig.RuntimeSettings.BasePath = serverService.BasePath();
             configService.SaveConfig(serverConfig);
 
+            if (config.flaresolverrurl != serverConfig.FlareSolverrUrl)
+            {
+                serverConfig.FlareSolverrUrl = config.flaresolverrurl;
+                configService.SaveConfig(serverConfig);
+                webHostRestartNeeded = true;
+            }
+
             if (omdbApiKey != serverConfig.OmdbApiKey || omdbApiUrl != serverConfig.OmdbApiUrl)
             {
                 serverConfig.OmdbApiKey = omdbApiKey;

src/Jackett.Server/Services/ServerService.cs

@@ -139,6 +139,8 @@ namespace Jackett.Server.Services
 
                 logger.Info($"Using proxy: {config.ProxyType}");
 
+                logger.Info("Using FlareSolverr: " + (string.IsNullOrEmpty(config.FlareSolverrUrl) ? "No" : config.FlareSolverrUrl));
+
                 var monotype = Type.GetType("Mono.Runtime");
                 if (monotype != null && !DotNetCoreUtil.IsRunningOnDotNetCore)
                 {