Rewrite shizaproject indexer to use graphql api without graphql client (#11715)

src/Jackett.Common/Indexers/ShizaProject.cs

@@ -1,19 +1,16 @@
 using System;
 using System.Collections.Generic;
-using System.Collections.Specialized;
 using System.Diagnostics.CodeAnalysis;
+using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
-using AngleSharp.Dom;
-using AngleSharp.Html.Parser;
 using Jackett.Common.Models;
 using Jackett.Common.Models.IndexerConfig;
 using Jackett.Common.Services.Interfaces;
-using Jackett.Common.Utils;
 using Jackett.Common.Utils.Clients;
+using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using NLog;
-
 namespace Jackett.Common.Indexers
 {
     [ExcludeFromCodeCoverage]
@@ -41,11 +38,11 @@ namespace Jackett.Common.Indexers
                    logger: l,
                    p: ps,
                    cacheService: cs,
-                   configData: new ConfigurationDataBasicLoginWithEmail())
+                   configData: new ConfigurationData())
         {
             Encoding = Encoding.UTF8;
             Language = "ru-ru";
-            Type = "semi-private";
+            Type = "public";
 
             AddCategoryMapping(1, TorznabCatType.TVAnime, "Anime");
         }
@@ -53,168 +50,222 @@ namespace Jackett.Common.Indexers
         private ConfigurationDataBasicLoginWithEmail Configuration => (ConfigurationDataBasicLoginWithEmail)configData;
 
         /// <summary>
-        /// http://shiza-project.com/accounts/login
+        /// http://shiza-project.com/graphql
         /// </summary>
-        private string LoginUrl => SiteLink + "accounts/login";
+        private string GraphqlEndpointUrl => SiteLink + "graphql";
-
-        /// <summary>
-        /// http://shiza-project.com/releases/search
-        /// </summary>
-        private string SearchUrl => SiteLink + "releases/search";
 
         public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
         {
             LoadValuesFromJson(configJson);
+            var releases = await PerformQuery(new TorznabQuery());
 
-            var data = new Dictionary<string, string>
+            await ConfigureIfOK(string.Empty, releases.Any(), () =>
-            {
+                throw new Exception("Could not find releases from this URL"));
-                { "field-email", Configuration.Email.Value },
-                { "field-password", Configuration.Password.Value }
-            };
-
-            var result = await RequestLoginAndFollowRedirect(
-                LoginUrl,
-                data,
-                null,
-                returnCookiesFromFirstCall: true
-            );
-
-            var parser = new HtmlParser();
-            var document = await parser.ParseDocumentAsync(result.ContentString);
-
-            await ConfigureIfOK(result.Cookies, IsAuthorized(result), () =>
-            {
-                var errorMessage = document.QuerySelector("div.alert-error").Text().Trim();
-                throw new ExceptionWithConfigData(errorMessage, Configuration);
-            });
 
             return IndexerConfigurationStatus.Completed;
         }
 
-        public override async Task<byte[]> Download(Uri link)
-        {
-            await EnsureAuthorized();
-            return await base.Download(link);
-        }
-
-        // If the search string is empty use the latest releases
         protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
         {
-            await EnsureAuthorized();
+            var releasesQuery = new
-
-            WebResult result;
-            if (query.IsTest || string.IsNullOrWhiteSpace(query.SearchTerm))
             {
-                result = await RequestWithCookiesAndRetryAsync(SiteLink);
+                operationName = "fetchReleases",
+                variables = new
+                {
+                    first = 50, //Number of fetched releases (required parameter) TODO: consider adding pagination 
+                    query = query.SearchTerm
+                },
+                query = @"
+                query fetchReleases($first: Int, $query: String) {
+                    releases(first: $first, query: $query) {
+                        edges {
+                            node {
+                                name
+                                originalName
+                                alternativeNames
+                                publishedAt
+                                slug
+                                posters {
+                                    preview: resize(width: 360, height: 500) {
+                                        url
                                     }
-            else
+                                }
-            {
+                                torrents {
-                // Prepare the search query
+                                    downloaded
-                var queryParameters = new NameValueCollection
+                                    seeders
-                {
+                                    leechers
-                    { "q", query.SearchTerm}
+                                    size
+                                    magnetUri
+                                    updatedAt
+                                    file {
+                                        url
+                                    }
+                                    videoQualities
+                                }
+                            }
+                        }
+                    }
+                }"
             };
-                result = await RequestWithCookiesAndRetryAsync(SearchUrl + "?" + queryParameters.GetQueryString());
+            var headers = new Dictionary<string, string>
-            }
+            {
-
+                { "Content-Type", "application/json; charset=utf-8" },
-            const string ReleaseLinksSelector = "article.grid-card > a.card-box";
+            };
+            var response = await RequestWithCookiesAndRetryAsync(GraphqlEndpointUrl, method: RequestType.POST, rawbody: Newtonsoft.Json.JsonConvert.SerializeObject(releasesQuery), headers: headers);
+            var j = JsonConvert.DeserializeObject<ReleasesResponse>(response.ContentString);
             var releases = new List<ReleaseInfo>();
-
+            foreach (var e in j.Data.Releases.Edges)
-            try
             {
-                var parser = new HtmlParser();
+                var n = e.Node;
-                var document = await parser.ParseDocumentAsync(result.ContentString);
-
-                foreach (var linkNode in document.QuerySelectorAll(ReleaseLinksSelector))
-                {
-                    var url = linkNode.GetAttribute("href");
-                    releases.AddRange(await FetchShowReleases(url));
-                }
-            }
-            catch (Exception ex)
-            {
-                OnParseError(result.ContentString, ex);
-            }
-
-            return releases;
-        }
-
-        private async Task EnsureAuthorized()
-        {
-            var result = await RequestWithCookiesAndRetryAsync(SiteLink);
-
-            if (!IsAuthorized(result))
-            {
-                await ApplyConfiguration(null);
-            }
-        }
-
-        private async Task<List<ReleaseInfo>> FetchShowReleases(string url)
-        {
-            var releases = new List<ReleaseInfo>();
-            var uri = new Uri(url);
-            var result = await RequestWithCookiesAndRetryAsync(url);
-
-            try
-            {
-                var parser = new HtmlParser();
-                var document = await parser.ParseDocumentAsync(result.ContentString);
-                var r = document.QuerySelector("div.release > div.wrapper-release");
-
                 var baseRelease = new ReleaseInfo
                 {
-                    Title = composeBaseTitle(r),
+                    Title = composeTitle(n),
-                    Poster = new Uri(SiteLink + r.QuerySelector("a[data-fancybox]").Attributes["href"].Value),
+                    Poster = n.Posters[0].Preview.Url,
-                    Details = uri,
+                    Details = new Uri(SiteLink + "releases/" + n.Slug),
                     DownloadVolumeFactor = 0,
                     UploadVolumeFactor = 1,
                     Category = new[] { TorznabCatType.TVAnime.ID }
                 };
 
-                foreach (var t in r.QuerySelectorAll("a[data-toggle]"))
+                foreach (var t in n.Torrents)
                 {
                     var release = (ReleaseInfo)baseRelease.Clone();
-                    release.Title += " " + t.Text().Trim();
+
-                    var tr_id = t.Attributes["href"].Value;
+                    release.Title += getTitleQualities(t);
-                    var tr = r.QuerySelector("div" + tr_id);
+                    release.Size = t.Size;
-                    release.Link = new Uri(tr.QuerySelector("a.button--success").Attributes["href"].Value);
+                    release.Seeders = t.Seeders;
-                    release.Seeders = long.Parse(tr.QuerySelector("div.torrent-counter > div:nth-of-type(1)").Text().Trim().Split(' ')[0]);
+                    release.Peers = t.Leechers + t.Seeders;
-                    release.Peers = release.Seeders + long.Parse(tr.QuerySelector("div.torrent-counter > div:nth-of-type(2)").Text().Trim().Split(' ')[0]);
+                    release.Grabs = t.Downloaded;
-                    release.Grabs = long.Parse(tr.QuerySelector("div.torrent-counter > div:nth-of-type(3)").Text().Trim().Split(' ')[0]);
+                    release.Link = t.File.Url;
-                    release.PublishDate = DateTime.Parse(tr.QuerySelector("time.torrent-time").Text());
+                    release.Guid = t.File.Url;
-                    release.Size = getReleaseSize(tr);
+                    release.MagnetUri = t.MagnetUri;
-                    release.Guid = new Uri(uri.ToString() + tr_id);
+                    release.PublishDate = getActualPublishDate(n, t);
                     releases.Add(release);
                 }
             }
-            catch (Exception ex)
-            {
-                OnParseError(result.ContentString, ex);
-            }
 
             return releases;
         }
 
-        private string composeBaseTitle(IElement release)
+        private string composeTitle(Node n)
         {
-            var titleDiv = release.QuerySelector("section:nth-of-type(2) > div.card > article:nth-of-type(1) > div.card-header");
+            var title = n.Name;
-            return titleDiv.QuerySelector("h3").Text() + " " + titleDiv.QuerySelector("p").Text();
+            title += " / " + n.OriginalName;
+            foreach (string name in n.AlternativeNames)
+                title += " / " + name;
+
+            return title;
         }
 
-        // Appending id to differentiate between different quality versions
+        private DateTime getActualPublishDate(Node n, Torrent t)
-        private bool IsAuthorized(WebResult result)
         {
-            return result.ContentString.Contains("/logout");
+            if (n.PublishedAt == null)
+            {
+                return t.UpdatedAt;
+            }
+            else
+            {
+                return (t.UpdatedAt > n.PublishedAt) ? t.UpdatedAt : n.PublishedAt.Value;
+            }
         }
 
-        private static long getReleaseSize(IElement tr)
+        private string getTitleQualities(Torrent t)
         {
-            var size = tr.QuerySelector("a.torrent-size").Text().Trim();
+            var s = " [";
-            size = size.Replace("КБ", "KB");
+
-            size = size.Replace("МБ", "MB");
+            foreach (string q in t.VideoQualities)
-            size = size.Replace("ГБ", "GB");
+            {
-            size = size.Replace("ТБ", "TB");
+                s += " " + q;
-            return ReleaseInfo.GetBytes(size);
+            }
+
+            return s + " ]";
+        }
+
+        public partial class ReleasesResponse
+        {
+            [JsonProperty("data")]
+            public Data Data { get; set; }
+        }
+
+        public partial class Data
+        {
+            [JsonProperty("releases")]
+            public Releases Releases { get; set; }
+        }
+
+        public partial class Releases
+        {
+            [JsonProperty("edges")]
+            public Edge[] Edges { get; set; }
+        }
+
+        public partial class Edge
+        {
+            [JsonProperty("node")]
+            public Node Node { get; set; }
+        }
+
+        public partial class Node
+        {
+            [JsonProperty("name")]
+            public string Name { get; set; }
+
+            [JsonProperty("originalName")]
+            public string OriginalName { get; set; }
+
+            [JsonProperty("alternativeNames")]
+            public string[] AlternativeNames { get; set; }
+
+            [JsonProperty("publishedAt")]
+            public DateTime? PublishedAt { get; set; }
+
+            [JsonProperty("slug")]
+            public string Slug { get; set; }
+
+            [JsonProperty("posters")]
+            public Poster[] Posters { get; set; }
+
+            [JsonProperty("torrents")]
+            public Torrent[] Torrents { get; set; }
+        }
+
+        public partial class Poster
+        {
+            [JsonProperty("preview")]
+            public Preview Preview { get; set; }
+        }
+
+        public partial class Preview
+        {
+            [JsonProperty("url")]
+            public Uri Url { get; set; }
+        }
+
+        public partial class Torrent
+        {
+            [JsonProperty("downloaded")]
+            public long Downloaded { get; set; }
+
+            [JsonProperty("seeders")]
+            public long Seeders { get; set; }
+
+            [JsonProperty("leechers")]
+            public long Leechers { get; set; }
+
+            [JsonProperty("size")]
+            public long Size { get; set; }
+
+            [JsonProperty("magnetUri")]
+            public Uri MagnetUri { get; set; }
+
+            [JsonProperty("updatedAt")]
+            public DateTime UpdatedAt { get; set; }
+
+            [JsonProperty("file")]
+            public Preview File { get; set; }
+
+            [JsonProperty("videoQualities")]
+            public string[] VideoQualities { get; set; }
         }
     }
 }