New: (Gazelle/OPS/RED) Prevent downloads without FL tokens

src/NzbDrone.Core/Datastore/Migration/041_gazelle_freeleech_token_options.cs

@@ -0,0 +1,60 @@
+using System.Collections.Generic;
+using System.Data;
+using Dapper;
+using FluentMigrator;
+using Newtonsoft.Json.Linq;
+using NzbDrone.Common.Serializer;
+using NzbDrone.Core.Datastore.Migration.Framework;
+
+namespace NzbDrone.Core.Datastore.Migration
+{
+    [Migration(041)]
+    public class gazelle_freeleech_token_options : NzbDroneMigrationBase
+    {
+        protected override void MainDbUpgrade()
+        {
+            Execute.WithConnection(MigrateIndexersToTokenOptions);
+        }
+
+        private void MigrateIndexersToTokenOptions(IDbConnection conn, IDbTransaction tran)
+        {
+            var updated = new List<object>();
+
+            using (var cmd = conn.CreateCommand())
+            {
+                cmd.Transaction = tran;
+                cmd.CommandText = "SELECT \"Id\", \"Settings\" FROM \"Indexers\" WHERE \"Implementation\" IN ('Orpheus', 'Redacted', 'AlphaRatio', 'BrokenStones', 'CGPeers', 'DICMusic', 'GreatPosterWall', 'SecretCinema')";
+
+                using (var reader = cmd.ExecuteReader())
+                {
+                    while (reader.Read())
+                    {
+                        var id = reader.GetInt32(0);
+                        var settings = Json.Deserialize<JObject>(reader.GetString(1));
+
+                        if (settings.ContainsKey("useFreeleechToken") && settings.Value<JToken>("useFreeleechToken").Type == JTokenType.Boolean)
+                        {
+                            var optionValue = settings.Value<bool>("useFreeleechToken") switch
+                            {
+                                true => 2, // Required
+                                _ => 0 // Never
+                            };
+
+                            settings.Remove("useFreeleechToken");
+                            settings.Add("useFreeleechToken", optionValue);
+                        }
+
+                        updated.Add(new
+                        {
+                            Id = id,
+                            Settings = settings.ToJson()
+                        });
+                    }
+                }
+            }
+
+            var updateSql = "UPDATE \"Indexers\" SET \"Settings\" = @Settings WHERE \"Id\" = @Id";
+            conn.Execute(updateSql, updated, transaction: tran);
+        }
+    }
+}

src/NzbDrone.Core/Indexers/Definitions/AlphaRatio.cs

@@ -143,7 +143,7 @@ public class AlphaRatioParser : GazelleParser
             .AddQueryParam("action", "download")
             .AddQueryParam("id", torrentId);
 
-        if (Settings.UseFreeleechToken && canUseToken)
+        if (Settings.UseFreeleechToken is (int)GazelleFreeleechTokenAction.Preferred or (int)GazelleFreeleechTokenAction.Required && canUseToken)
         {
             url = url.AddQueryParam("usetoken", "1");
         }

src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleBase.cs

@@ -4,6 +4,7 @@ using System.Linq;
 using System.Net.Http;
 using System.Threading.Tasks;
 using NLog;
+using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.Messaging.Events;
@@ -85,22 +86,23 @@ public abstract class GazelleBase<TSettings> : TorrentIndexerBase<TSettings>
     public override async Task<IndexerDownloadResponse> Download(Uri link)
     {
         var downloadResponse = await base.Download(link);
-        var response = downloadResponse.Data;
 
-        if (response.Length >= 1
+        var fileData = downloadResponse.Data;
-            && response[0] != 'd' // simple test for torrent vs HTML content
+
+        if (Settings.UseFreeleechToken == (int)GazelleFreeleechTokenAction.Preferred
+            && fileData.Length >= 1
+            && fileData[0] != 'd' // simple test for torrent vs HTML content
             && link.Query.Contains("usetoken=1"))
         {
-            var html = Encoding.GetString(response);
+            var html = Encoding.GetString(fileData);
+
             if (html.Contains("You do not have any freeleech tokens left.")
                 || html.Contains("You do not have enough freeleech tokens")
                 || html.Contains("This torrent is too large.")
                 || html.Contains("You cannot use tokens here"))
             {
-                // download again without usetoken=1
+                // Try to download again without usetoken
-                var requestLinkNew = link.ToString().Replace("&usetoken=1", "");
+                downloadResponse = await base.Download(link.RemoveQueryParam("usetoken"));
-
-                downloadResponse = await base.Download(new Uri(requestLinkNew));
             }
         }
 

src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleParser.cs

@@ -62,7 +62,7 @@ public class GazelleParser : IParseIndexerResponse
                     var isFreeLeech = torrent.IsFreeLeech || torrent.IsNeutralLeech || torrent.IsPersonalFreeLeech;
 
                     // skip releases that cannot be used with freeleech tokens when the option is enabled
-                    if (Settings.UseFreeleechToken && !torrent.CanUseToken && !isFreeLeech)
+                    if (Settings.UseFreeleechToken == (int)GazelleFreeleechTokenAction.Required && !torrent.CanUseToken && !isFreeLeech)
                     {
                         continue;
                     }
@@ -115,7 +115,7 @@ public class GazelleParser : IParseIndexerResponse
                 var isFreeLeech = result.IsFreeLeech || result.IsNeutralLeech || result.IsPersonalFreeLeech;
 
                 // skip releases that cannot be used with freeleech tokens when the option is enabled
-                if (Settings.UseFreeleechToken && !result.CanUseToken && !isFreeLeech)
+                if (Settings.UseFreeleechToken == (int)GazelleFreeleechTokenAction.Required && !result.CanUseToken && !isFreeLeech)
                 {
                     continue;
                 }
@@ -169,7 +169,7 @@ public class GazelleParser : IParseIndexerResponse
             .AddQueryParam("action", "download")
             .AddQueryParam("id", torrentId);
 
-        if (Settings.UseFreeleechToken)
+        if (Settings.UseFreeleechToken is (int)GazelleFreeleechTokenAction.Preferred or (int)GazelleFreeleechTokenAction.Required && canUseToken)
         {
             url = url.AddQueryParam("usetoken", "1");
         }

src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleSettings.cs

@@ -16,11 +16,23 @@ public class GazelleSettings : UserPassTorrentBaseSettings
     public string AuthKey { get; set; }
     public string PassKey { get; set; }
 
-    [FieldDefinition(5, Type = FieldType.Checkbox, Label = "Use Freeleech Token", HelpText = "Use freeleech tokens when available")]
+    [FieldDefinition(5, Type = FieldType.Select, Label = "Use Freeleech Tokens", SelectOptions = typeof(GazelleFreeleechTokenAction), HelpText = "When to use freeleech tokens")]
-    public bool UseFreeleechToken { get; set; }
+    public int UseFreeleechToken { get; set; }
 
     public override NzbDroneValidationResult Validate()
     {
         return new NzbDroneValidationResult(Validator.Validate(this));
     }
 }
+
+public enum GazelleFreeleechTokenAction
+{
+    [FieldOption(Label = "Never", Hint = "Do not use tokens")]
+    Never = 0,
+
+    [FieldOption(Label = "Preferred", Hint = "Use token if possible")]
+    Preferred = 1,
+
+    [FieldOption(Label = "Required", Hint = "Abort download if unable to use token")]
+    Required = 2,
+}

src/NzbDrone.Core/Indexers/Definitions/GreatPosterWall.cs

@@ -171,7 +171,7 @@ public class GreatPosterWallParser : GazelleParser
                 var isFreeLeech = torrent.IsFreeleech || torrent.IsNeutralLeech || torrent.IsPersonalFreeleech;
 
                 // skip releases that cannot be used with freeleech tokens when the option is enabled
-                if (_settings.UseFreeleechToken && !torrent.CanUseToken && !isFreeLeech)
+                if (_settings.UseFreeleechToken == (int)GazelleFreeleechTokenAction.Required && !torrent.CanUseToken && !isFreeLeech)
                 {
                     continue;
                 }
@@ -242,7 +242,7 @@ public class GreatPosterWallParser : GazelleParser
             .AddQueryParam("action", "download")
             .AddQueryParam("id", torrentId);
 
-        if (_settings.UseFreeleechToken && canUseToken)
+        if (_settings.UseFreeleechToken is (int)GazelleFreeleechTokenAction.Preferred or (int)GazelleFreeleechTokenAction.Required && canUseToken)
         {
             url = url.AddQueryParam("usetoken", "1");
         }

src/NzbDrone.Core/Indexers/Definitions/Orpheus.cs

@@ -50,6 +50,20 @@ namespace NzbDrone.Core.Indexers.Definitions
             return new OrpheusParser(Settings, Capabilities.Categories);
         }
 
+        protected override Task<HttpRequest> GetDownloadRequest(Uri link)
+        {
+            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri)
+            {
+                AllowAutoRedirect = FollowRedirect
+            };
+
+            var request = requestBuilder
+                .SetHeader("Authorization", $"token {Settings.Apikey}")
+                .Build();
+
+            return Task.FromResult(request);
+        }
+
         protected override IList<ReleaseInfo> CleanupReleases(IEnumerable<ReleaseInfo> releases, SearchCriteriaBase searchCriteria)
         {
             var cleanReleases = base.CleanupReleases(releases, searchCriteria);
@@ -91,47 +105,28 @@ namespace NzbDrone.Core.Indexers.Definitions
 
         public override async Task<IndexerDownloadResponse> Download(Uri link)
         {
-            var request = new HttpRequestBuilder(link.AbsoluteUri)
+            var downloadResponse = await base.Download(link);
-                .SetHeader("Authorization", $"token {Settings.Apikey}")
-                .Build();
 
-            byte[] downloadBytes;
+            var fileData = downloadResponse.Data;
-            long elapsedTime;
 
-            try
+            if (Settings.UseFreeleechToken == (int)OrpheusFreeleechTokenAction.Preferred
+                && fileData.Length >= 1
+                && fileData[0] != 'd' // simple test for torrent vs HTML content
+                && link.Query.Contains("usetoken=1"))
             {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
+                var html = Encoding.GetString(fileData);
-                downloadBytes = response.ResponseData;
-                elapsedTime = response.ElapsedTime;
 
-                if (downloadBytes.Length >= 1
+                if (html.Contains("You do not have any freeleech tokens left.")
-                    && downloadBytes[0] != 'd' // simple test for torrent vs HTML content
+                    || html.Contains("You do not have enough freeleech tokens")
-                    && link.Query.Contains("usetoken=1"))
+                    || html.Contains("This torrent is too large.")
+                    || html.Contains("You cannot use tokens here"))
                 {
-                    var html = Encoding.GetString(downloadBytes);
+                    // Try to download again without usetoken
-                    if (html.Contains("You do not have any freeleech tokens left.")
+                    downloadResponse = await base.Download(link.RemoveQueryParam("usetoken"));
-                        || html.Contains("You do not have enough freeleech tokens")
-                        || html.Contains("This torrent is too large.")
-                        || html.Contains("You cannot use tokens here"))
-                    {
-                        // download again without usetoken
-                        request.Url = new HttpUri(link.ToString().Replace("&usetoken=1", ""));
-
-                        response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                        downloadBytes = response.ResponseData;
-                    }
                 }
             }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Download failed");
-                throw;
-            }
 
-            ValidateDownloadData(downloadBytes);
+            return downloadResponse;
-
-            return new IndexerDownloadResponse(downloadBytes, elapsedTime);
         }
     }
 
@@ -280,7 +275,7 @@ namespace NzbDrone.Core.Indexers.Definitions
                     foreach (var torrent in result.Torrents)
                     {
                         // skip releases that cannot be used with freeleech tokens when the option is enabled
-                        if (_settings.UseFreeleechToken && !torrent.CanUseToken)
+                        if (_settings.UseFreeleechToken == (int)OrpheusFreeleechTokenAction.Required && !torrent.CanUseToken)
                         {
                             continue;
                         }
@@ -331,7 +326,7 @@ namespace NzbDrone.Core.Indexers.Definitions
                 else
                 {
                     // skip releases that cannot be used with freeleech tokens when the option is enabled
-                    if (_settings.UseFreeleechToken && !result.CanUseToken)
+                    if (_settings.UseFreeleechToken == (int)OrpheusFreeleechTokenAction.Required && !result.CanUseToken)
                     {
                         continue;
                     }
@@ -418,7 +413,7 @@ namespace NzbDrone.Core.Indexers.Definitions
                 .AddQueryParam("id", torrentId);
 
             // Orpheus fails to download if usetoken=0 so we need to only add if we will use one
-            if (_settings.UseFreeleechToken && canUseToken)
+            if (_settings.UseFreeleechToken is (int)OrpheusFreeleechTokenAction.Preferred or (int)OrpheusFreeleechTokenAction.Required && canUseToken)
             {
                 url = url.AddQueryParam("usetoken", "1");
             }
@@ -452,18 +447,30 @@ namespace NzbDrone.Core.Indexers.Definitions
         public OrpheusSettings()
         {
             Apikey = "";
-            UseFreeleechToken = false;
+            UseFreeleechToken = (int)OrpheusFreeleechTokenAction.Never;
         }
 
         [FieldDefinition(2, Label = "ApiKey", HelpText = "IndexerOrpheusSettingsApiKeyHelpText", Privacy = PrivacyLevel.ApiKey)]
         public string Apikey { get; set; }
 
-        [FieldDefinition(3, Label = "Use Freeleech Tokens", HelpText = "Use freeleech tokens when available", Type = FieldType.Checkbox)]
+        [FieldDefinition(3, Type = FieldType.Select, Label = "Use Freeleech Tokens", SelectOptions = typeof(OrpheusFreeleechTokenAction), HelpText = "When to use freeleech tokens")]
-        public bool UseFreeleechToken { get; set; }
+        public int UseFreeleechToken { get; set; }
 
         public override NzbDroneValidationResult Validate()
         {
             return new NzbDroneValidationResult(Validator.Validate(this));
         }
     }
+
+    public enum OrpheusFreeleechTokenAction
+    {
+        [FieldOption(Label = "Never", Hint = "Do not use tokens")]
+        Never = 0,
+
+        [FieldOption(Label = "Preferred", Hint = "Use token if possible")]
+        Preferred = 1,
+
+        [FieldOption(Label = "Required", Hint = "Abort download if unable to use token")]
+        Required = 2,
+    }
 }

src/NzbDrone.Core/Indexers/Definitions/Redacted.cs

@@ -102,6 +102,32 @@ namespace NzbDrone.Core.Indexers.Definitions
 
             return caps;
         }
+
+        public override async Task<IndexerDownloadResponse> Download(Uri link)
+        {
+            var downloadResponse = await base.Download(link);
+
+            var fileData = downloadResponse.Data;
+
+            if (Settings.UseFreeleechToken == (int)RedactedFreeleechTokenAction.Preferred
+                && fileData.Length >= 1
+                && fileData[0] != 'd' // simple test for torrent vs HTML content
+                && link.Query.Contains("usetoken=1"))
+            {
+                var html = Encoding.GetString(fileData);
+
+                if (html.Contains("You do not have any freeleech tokens left.")
+                    || html.Contains("You do not have enough freeleech tokens")
+                    || html.Contains("This torrent is too large.")
+                    || html.Contains("You cannot use tokens here"))
+                {
+                    // Try to download again without usetoken
+                    downloadResponse = await base.Download(link.RemoveQueryParam("usetoken"));
+                }
+            }
+
+            return downloadResponse;
+        }
     }
 
     public class RedactedRequestGenerator : IIndexerRequestGenerator
@@ -248,7 +274,7 @@ namespace NzbDrone.Core.Indexers.Definitions
                     foreach (var torrent in result.Torrents)
                     {
                         // skip releases that cannot be used with freeleech tokens when the option is enabled
-                        if (_settings.UseFreeleechToken && !torrent.CanUseToken)
+                        if (_settings.UseFreeleechToken == (int)RedactedFreeleechTokenAction.Required && !torrent.CanUseToken)
                         {
                             continue;
                         }
@@ -305,7 +331,7 @@ namespace NzbDrone.Core.Indexers.Definitions
                 else
                 {
                     // skip releases that cannot be used with freeleech tokens when the option is enabled
-                    if (_settings.UseFreeleechToken && !result.CanUseToken)
+                    if (_settings.UseFreeleechToken == (int)RedactedFreeleechTokenAction.Required && !result.CanUseToken)
                     {
                         continue;
                     }
@@ -397,7 +423,7 @@ namespace NzbDrone.Core.Indexers.Definitions
                 .AddQueryParam("action", "download")
                 .AddQueryParam("id", torrentId);
 
-            if (_settings.UseFreeleechToken && canUseToken)
+            if (_settings.UseFreeleechToken is (int)RedactedFreeleechTokenAction.Preferred or (int)RedactedFreeleechTokenAction.Required && canUseToken)
             {
                 url = url.AddQueryParam("usetoken", "1");
             }
@@ -431,14 +457,14 @@ namespace NzbDrone.Core.Indexers.Definitions
         public RedactedSettings()
         {
             Apikey = "";
-            UseFreeleechToken = false;
+            UseFreeleechToken = (int)RedactedFreeleechTokenAction.Never;
         }
 
         [FieldDefinition(2, Label = "ApiKey", Privacy = PrivacyLevel.ApiKey, HelpText = "IndexerRedactedSettingsApiKeyHelpText")]
         public string Apikey { get; set; }
 
-        [FieldDefinition(3, Label = "Use Freeleech Tokens", Type = FieldType.Checkbox, HelpText = "Use freeleech tokens when available")]
+        [FieldDefinition(3, Type = FieldType.Select, Label = "Use Freeleech Tokens", SelectOptions = typeof(RedactedFreeleechTokenAction), HelpText = "When to use freeleech tokens")]
-        public bool UseFreeleechToken { get; set; }
+        public int UseFreeleechToken { get; set; }
 
         [FieldDefinition(4, Label = "Freeload Only", Type = FieldType.Checkbox, Advanced = true, HelpTextWarning = "Search freeload torrents only. End date: 31 January 2024, 23:59 UTC.")]
         public bool FreeloadOnly { get; set; }
@@ -448,4 +474,16 @@ namespace NzbDrone.Core.Indexers.Definitions
             return new NzbDroneValidationResult(Validator.Validate(this));
         }
     }
+
+    public enum RedactedFreeleechTokenAction
+    {
+        [FieldOption(Label = "Never", Hint = "Do not use tokens")]
+        Never = 0,
+
+        [FieldOption(Label = "Preferred", Hint = "Use token if possible")]
+        Preferred = 1,
+
+        [FieldOption(Label = "Required", Hint = "Abort download if unable to use token")]
+        Required = 2,
+    }
 }

src/NzbDrone.Core/Indexers/Definitions/SecretCinema.cs

@@ -226,7 +226,7 @@ public class SecretCinemaParser : IParseIndexerResponse
             .AddQueryParam("action", "download")
             .AddQueryParam("id", torrentId);
 
-        if (_settings.UseFreeleechToken)
+        if (_settings.UseFreeleechToken is (int)GazelleFreeleechTokenAction.Preferred or (int)GazelleFreeleechTokenAction.Required)
         {
             url = url.AddQueryParam("useToken", "1");
         }