beyondhdapi: refactor and assume UTC for publish dates (#15627)

src/Jackett.Common/Indexers/Definitions/BeyondHDAPI.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
+using System.Globalization;
 using System.Linq;
 using System.Threading.Tasks;
 using Jackett.Common.Extensions;
@@ -29,7 +30,8 @@ namespace Jackett.Common.Indexers.Definitions
 
         public override TorznabCapabilities TorznabCaps => SetCapabilities();
 
-        private readonly string APIBASE = "https://beyond-hd.me/api/torrents/";
+        protected virtual int ApiKeyLength => 32;
+        protected virtual int RSSKeyLength => 32;
 
         private new ConfigurationDataBeyondHDApi configData
         {
@@ -69,8 +71,17 @@ namespace Jackett.Common.Indexers.Definitions
 
             return caps;
         }
-        protected virtual int ApiKeyLength => 32;
+
-        protected virtual int RSSKeyLength => 32;
+        public override IIndexerRequestGenerator GetRequestGenerator()
+        {
+            return new BeyondHDAPIRequestGenerator(configData, TorznabCaps);
+        }
+
+        public override IParseIndexerResponse GetParser()
+        {
+            return new BeyondHDAPIParser(configData, TorznabCaps.Categories, logger);
+        }
+
         public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
         {
             LoadValuesFromJson(configJson);
@@ -112,45 +123,58 @@ namespace Jackett.Common.Indexers.Definitions
 
             return IndexerConfigurationStatus.Completed;
         }
+    }
 
-        protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
+    public class BeyondHDAPIRequestGenerator : IIndexerRequestGenerator
     {
-            var apiKey = configData.ApiKey.Value;
+        private readonly ConfigurationDataBeyondHDApi _configData;
-            var apiUrl = $"{APIBASE}{apiKey}";
+        private readonly TorznabCapabilities _capabilities;
+
+        private const string ApiBase = "https://beyond-hd.me/api/torrents/";
+
+        public BeyondHDAPIRequestGenerator(ConfigurationDataBeyondHDApi configData, TorznabCapabilities capabilities)
+        {
+            _configData = configData;
+            _capabilities = capabilities;
+        }
+
+        public IndexerPageableRequestChain GetSearchRequests(TorznabQuery query)
+        {
+            var pageableRequests = new IndexerPageableRequestChain();
 
             var postData = new Dictionary<string, object>
             {
                 { BHDParams.action, "search" },
-                { BHDParams.rsskey, configData.RSSKey.Value },
+                { BHDParams.rsskey, _configData.RSSKey.Value },
                 { BHDParams.search, query.GetQueryString() },
             };
 
-            if (configData.FilterFreeleech.Value)
+            if (_configData.FilterFreeleech.Value)
             {
                 postData.Add(BHDParams.freeleech, 1);
             }
 
-            if (configData.FilterLimited.Value)
+            if (_configData.FilterLimited.Value)
             {
                 postData.Add(BHDParams.limited, 1);
             }
 
-            if (configData.FilterRefund.Value)
+            if (_configData.FilterRefund.Value)
             {
                 postData.Add(BHDParams.refund, 1);
             }
 
-            if (configData.FilterRewind.Value)
+            if (_configData.FilterRewind.Value)
             {
                 postData.Add(BHDParams.rewind, 1);
             }
 
-            if (configData.SearchTypes.Values.Any())
+            if (_configData.SearchTypes.Values.Any())
             {
-                postData.Add(BHDParams.types, configData.SearchTypes.Values.ToArray());
+                postData.Add(BHDParams.types, _configData.SearchTypes.Values.ToArray());
             }
 
-            var categories = MapTorznabCapsToTrackers(query);
+            var categories = _capabilities.Categories.MapTorznabCapsToTrackers(query);
 
             if (categories.Any())
             {
@@ -172,110 +196,16 @@ namespace Jackett.Common.Indexers.Definitions
                 postData.Add("page", page.ToString());
             }
 
-            var bhdResponse = await GetBHDResponse(apiUrl, postData);
+            pageableRequests.Add(GetPagedRequests(postData));
 
-            var results = bhdResponse.Results
+            return pageableRequests;
-                                     .Where(r => r.DownloadUrl.IsNotNullOrWhiteSpace() && r.InfoUrl.IsNotNullOrWhiteSpace())
-                                     .ToList();
-
-            return results.Select(MapToReleaseInfo).ToList();
         }
 
-        private ReleaseInfo MapToReleaseInfo(BHDResult bhdResult)
+        private IEnumerable<IndexerRequest> GetPagedRequests(Dictionary<string, object> postData)
         {
-            var releaseInfo = new ReleaseInfo
+            var apiKey = _configData.ApiKey.Value;
-            {
+            var apiUrl = $"{ApiBase}{apiKey}";
-                Guid = new Uri(bhdResult.InfoUrl),
-                Details = new Uri(bhdResult.InfoUrl),
-                Link = new Uri(bhdResult.DownloadUrl),
-                Title = GetReleaseTitle(bhdResult),
-                Category = MapTrackerCatDescToNewznab(bhdResult.Category),
-                InfoHash = bhdResult.InfoHash,
-                Grabs = bhdResult.Grabs,
-                Size = bhdResult.Size,
-                Seeders = bhdResult.Seeders,
-                Peers = bhdResult.Leechers + bhdResult.Seeders,
-                PublishDate = bhdResult.CreatedAt
-            };
 
-            if (bhdResult.ImdbId.IsNotNullOrWhiteSpace())
-            {
-                releaseInfo.Imdb = ParseUtil.GetImdbId(bhdResult.ImdbId);
-            }
-
-            if (bhdResult.TmdbId.IsNotNullOrWhiteSpace())
-            {
-                var tmdbId = bhdResult.TmdbId.Split('/').ElementAtOrDefault(1);
-                releaseInfo.TMDb = tmdbId != null && ParseUtil.TryCoerceInt(tmdbId, out var tmdbResult) ? tmdbResult : 0;
-            }
-
-            releaseInfo.DownloadVolumeFactor = 1;
-            releaseInfo.UploadVolumeFactor = 1;
-
-            if (bhdResult.Freeleech == 1 || bhdResult.Limited == 1)
-            {
-                releaseInfo.DownloadVolumeFactor = 0;
-            }
-
-            if (bhdResult.Promo25 == 1)
-            {
-                releaseInfo.DownloadVolumeFactor = .75;
-            }
-
-            if (bhdResult.Promo50 == 1)
-            {
-                releaseInfo.DownloadVolumeFactor = .50;
-            }
-
-            if (bhdResult.Promo75 == 1)
-            {
-                releaseInfo.DownloadVolumeFactor = .25;
-            }
-
-            return releaseInfo;
-        }
-
-        private string GetReleaseTitle(BHDResult bhdResult)
-        {
-            var title = bhdResult.Name.Trim();
-
-            if (!configData.AddHybridFeaturesToTitle.Value)
-            {
-                return title;
-            }
-
-            var features = new List<string>();
-
-            if (bhdResult.DolbyVision == 1)
-            {
-                features.Add("Dolby Vision");
-            }
-
-            if (bhdResult.Hdr10 == 1)
-            {
-                features.Add("HDR10");
-            }
-
-            if (bhdResult.Hdr10Plus == 1)
-            {
-                features.Add("HDR10+");
-            }
-
-            if (bhdResult.Hlg == 1)
-            {
-                features.Add("HLG");
-            }
-
-            if (features.Count > 1)
-            {
-                title += $" ({string.Join(" / ", features)})";
-            }
-
-            return title;
-        }
-
-        private async Task<BHDResponse> GetBHDResponse(string apiUrl, Dictionary<string, object> postData)
-        {
             var request = new WebRequest
             {
                 Url = apiUrl,
@@ -288,22 +218,154 @@ namespace Jackett.Common.Indexers.Definitions
                 RawBody = JsonConvert.SerializeObject(postData)
             };
 
-            var response = await webclient.GetResultAsync(request);
+            yield return new IndexerRequest(request);
-            if (response != null && response.ContentString.StartsWith("<"))
+        }
+    }
+
+    public class BeyondHDAPIParser : IParseIndexerResponse
     {
-                // the response was not JSON, likely a HTML page for a server outage
+        private readonly ConfigurationDataBeyondHDApi _configData;
-                logger.Warn(response.ContentString);
+        private readonly TorznabCapabilitiesCategories _categories;
+        private readonly Logger _logger;
+
+        public BeyondHDAPIParser(ConfigurationDataBeyondHDApi configData, TorznabCapabilitiesCategories categories, Logger logger)
+        {
+            _configData = configData;
+            _categories = categories;
+            _logger = logger;
+        }
+
+        public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
+        {
+            var releases = new List<ReleaseInfo>();
+
+            if (indexerResponse.Content.StartsWith("<"))
+            {
+                // The response was not JSON, likely a HTML page for a server outage
+                _logger.Warn("The response was not JSON: {0}", indexerResponse.Content);
+
                 throw new Exception("The response was not JSON");
             }
 
-            var bhdresponse = JsonConvert.DeserializeObject<BHDResponse>(response.ContentString);
+            if (indexerResponse.Content.ContainsIgnoreCase("Invalid API Key"))
-
-            if (bhdresponse.status_code == 0)
             {
-                throw new Exception(bhdresponse.status_message);
+                throw new Exception("API Key invalid or not authorized");
             }
 
-            return bhdresponse;
+            var jsonResponse = JsonConvert.DeserializeObject<BHDResponse>(indexerResponse.Content);
+
+            if (jsonResponse.StatusCode == 0)
+            {
+                throw new Exception($"Indexer Error: {jsonResponse.StatusMessage}");
+            }
+
+            foreach (var row in jsonResponse.Results)
+            {
+                if (row.DownloadUrl.IsNullOrWhiteSpace() || row.InfoUrl.IsNullOrWhiteSpace())
+                {
+                    _logger.Warn("Missing download or info url for release with the Id {0}, skipping.", row.Id);
+                    continue;
+                }
+
+                // Skip invalid results when freeleech or limited filtering is set
+                if ((_configData.FilterFreeleech.Value && row.Freeleech == 0) || (_configData.FilterLimited.Value && row.Limited == 0))
+                {
+                    continue;
+                }
+
+                var releaseInfo = new ReleaseInfo
+                {
+                    Guid = new Uri(row.InfoUrl),
+                    Details = new Uri(row.InfoUrl),
+                    Link = new Uri(row.DownloadUrl),
+                    Title = GetReleaseTitle(row),
+                    Category = _categories.MapTrackerCatDescToNewznab(row.Category),
+                    InfoHash = row.InfoHash,
+                    Grabs = row.Grabs,
+                    Size = row.Size,
+                    Seeders = row.Seeders,
+                    Peers = row.Leechers + row.Seeders,
+                    PublishDate = DateTime.Parse(row.CreatedAt, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal)
+                };
+
+                if (row.ImdbId.IsNotNullOrWhiteSpace())
+                {
+                    releaseInfo.Imdb = ParseUtil.GetImdbId(row.ImdbId);
+                }
+
+                if (row.TmdbId.IsNotNullOrWhiteSpace())
+                {
+                    var tmdbId = row.TmdbId.Split('/').ElementAtOrDefault(1);
+                    releaseInfo.TMDb = tmdbId != null && ParseUtil.TryCoerceInt(tmdbId, out var tmdbResult) ? tmdbResult : 0;
+                }
+
+                releaseInfo.DownloadVolumeFactor = 1;
+                releaseInfo.UploadVolumeFactor = 1;
+
+                if (row.Freeleech == 1 || row.Limited == 1)
+                {
+                    releaseInfo.DownloadVolumeFactor = 0;
+                }
+
+                if (row.Promo25 == 1)
+                {
+                    releaseInfo.DownloadVolumeFactor = .75;
+                }
+
+                if (row.Promo50 == 1)
+                {
+                    releaseInfo.DownloadVolumeFactor = .50;
+                }
+
+                if (row.Promo75 == 1)
+                {
+                    releaseInfo.DownloadVolumeFactor = .25;
+                }
+
+                releases.Add(releaseInfo);
+            }
+
+            return releases;
+        }
+
+        private string GetReleaseTitle(BHDResult row)
+        {
+            var title = row.Name.Trim();
+
+            if (!_configData.AddHybridFeaturesToTitle.Value)
+            {
+                return title;
+            }
+
+            var features = new HashSet<string>();
+
+            if (row.DolbyVision == 1)
+            {
+                features.Add("Dolby Vision");
+            }
+
+            if (row.Hdr10 == 1)
+            {
+                features.Add("HDR10");
+            }
+
+            if (row.Hdr10Plus == 1)
+            {
+                features.Add("HDR10+");
+            }
+
+            if (row.Hlg == 1)
+            {
+                features.Add("HLG");
+            }
+
+            if (features.Any())
+            {
+                title += $" ({string.Join(" / ", features)})";
+            }
+
+            return title;
+        }
     }
 
     internal class BHDParams
@@ -368,13 +430,18 @@ namespace Jackett.Common.Indexers.Definitions
 
     class BHDResponse
     {
-            public int status_code { get; set; } // The status code of the post request. (0 = Failed and 1 = Success)
+        [JsonProperty("status_code")]
-            public string status_message { get; set; } // If status code=0 then there will be an explanation
+        public int StatusCode { get; set; } // The status code of the post request. (0 = Failed and 1 = Success)
+
+        [JsonProperty("status_message")]
+        public string StatusMessage { get; set; } // If status code=0 then there will be an explanation
+
+        public IReadOnlyCollection<BHDResult> Results { get; set; } // The results that match your query.
+
         public int page { get; set; } // The current page of results that you're on.
         public int total_pages { get; set; } // int The total number of pages of results matching your query.
         public int total_results { get; set; } // The total number of results matching your query.
         public bool success { get; set; } // The status of the call. (True = Success, False = Error)
-            public IReadOnlyCollection<BHDResult> Results { get; set; } // The results that match your query.
     }
 
     class BHDResult
@@ -417,7 +484,7 @@ namespace Jackett.Common.Indexers.Definitions
         public int Rescue { get; set; }
 
         [JsonProperty("created_at")]
-            public DateTime CreatedAt { get; set; }
+        public string CreatedAt { get; set; }
 
         [JsonProperty("url")]
         public string InfoUrl { get; set; }
@@ -435,5 +502,4 @@ namespace Jackett.Common.Indexers.Definitions
 
         public int Hlg { get; set; }
     }
-    }
 }

src/Jackett.Common/Models/IndexerConfig/Bespoke/ConfigurationDataBeyondHDApi.cs

@@ -5,7 +5,7 @@ using System.Diagnostics.CodeAnalysis;
 namespace Jackett.Common.Models.IndexerConfig.Bespoke
 {
     [ExcludeFromCodeCoverage]
-    internal class ConfigurationDataBeyondHDApi : ConfigurationData
+    public class ConfigurationDataBeyondHDApi : ConfigurationData
     {
         public StringConfigurationItem ApiKey { get; private set; }
         public StringConfigurationItem RSSKey { get; private set; }