mirror of
https://github.com/Jackett/Jackett.git
synced 2025-09-30 23:47:17 +02:00
* Replace Cardigann Anidex indexer for C# impelementation Add bypass for DDOS Guard * Improve error messages from type conversions * Add missing cookie check * Fix index out of range exception * Change error handling to only warn about DDoS bypass exceptions This is so that searches will still be attempted if there are issues with the DDoS protection (e.g. if it is removed). * Improve error handling and clean up code * pending changes
This commit is contained in:
Diego Heras
319
src/Jackett.Common/Indexers/Anidex.cs
Normal file
319
src/Jackett.Common/Indexers/Anidex.cs
Normal file
@@ -0,0 +1,319 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Specialized;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using AngleSharp.Dom;
|
||||
using AngleSharp.Html.Parser;
|
||||
using Jackett.Common.Models;
|
||||
using Jackett.Common.Models.IndexerConfig;
|
||||
using Jackett.Common.Services.Interfaces;
|
||||
using Jackett.Common.Utils;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using NLog;
|
||||
using static Jackett.Common.Models.IndexerConfig.ConfigurationData;
|
||||
|
||||
namespace Jackett.Common.Indexers
|
||||
{
|
||||
public class Anidex : BaseWebIndexer
|
||||
{
|
||||
public Anidex(IIndexerConfigurationService configService, Utils.Clients.WebClient wc, Logger l, IProtectionService ps)
|
||||
: base(name: "Anidex",
|
||||
description: "Anidex is a Public torrent tracker and indexer, primarily for English fansub groups of anime",
|
||||
link: "https://anidex.info/",
|
||||
caps: new TorznabCapabilities(),
|
||||
configService: configService,
|
||||
client: wc,
|
||||
logger: l,
|
||||
p: ps,
|
||||
configData: new ConfigurationData())
|
||||
{
|
||||
Encoding = Encoding.UTF8;
|
||||
Language = "en-us";
|
||||
Type = "public";
|
||||
|
||||
// Configure the category mappings
|
||||
AddCategoryMapping(1, TorznabCatType.TVAnime, "Anime - Sub");
|
||||
AddCategoryMapping(2, TorznabCatType.TVAnime, "Anime - Raw");
|
||||
AddCategoryMapping(3, TorznabCatType.TVAnime, "Anime - Dub");
|
||||
AddCategoryMapping(4, TorznabCatType.TVAnime, "LA - Sub");
|
||||
AddCategoryMapping(5, TorznabCatType.TVAnime, "LA - Raw");
|
||||
AddCategoryMapping(6, TorznabCatType.TVAnime, "Light Novel");
|
||||
AddCategoryMapping(7, TorznabCatType.TVAnime, "Manga - TLed");
|
||||
AddCategoryMapping(8, TorznabCatType.TVAnime, "Manga - Raw");
|
||||
AddCategoryMapping(9, TorznabCatType.TVAnime, "♫ - Lossy");
|
||||
AddCategoryMapping(10, TorznabCatType.TVAnime, "♫ - Lossless");
|
||||
AddCategoryMapping(11, TorznabCatType.TVAnime, "♫ - Video");
|
||||
AddCategoryMapping(12, TorznabCatType.TVAnime, "Games");
|
||||
AddCategoryMapping(13, TorznabCatType.TVAnime, "Applications");
|
||||
AddCategoryMapping(14, TorznabCatType.TVAnime, "Pictures");
|
||||
AddCategoryMapping(15, TorznabCatType.TVAnime, "Adult Video");
|
||||
AddCategoryMapping(16, TorznabCatType.TVAnime, "Other");
|
||||
|
||||
// Configure the language select option
|
||||
var languageSelect = new SelectItem(new Dictionary<string, string>()
|
||||
{
|
||||
{"1", "English"},
|
||||
{"2", "Japanese"},
|
||||
{"3", "Polish"},
|
||||
{"4", "Serbo-Croatian" },
|
||||
{"5", "Dutch"},
|
||||
{"6", "Italian"},
|
||||
{"7", "Russian"},
|
||||
{"8", "German"},
|
||||
{"9", "Hungarian"},
|
||||
{"10", "French"},
|
||||
{"11", "Finnish"},
|
||||
{"12", "Vietnamese"},
|
||||
{"13", "Greek"},
|
||||
{"14", "Bulgarian"},
|
||||
{"15", "Spanish (Spain)" },
|
||||
{"16", "Portuguese (Brazil)" },
|
||||
{"17", "Portuguese (Portugal)" },
|
||||
{"18", "Swedish"},
|
||||
{"19", "Arabic"},
|
||||
{"20", "Danish"},
|
||||
{"21", "Chinese (Simplified)" },
|
||||
{"22", "Bengali"},
|
||||
{"23", "Romanian"},
|
||||
{"24", "Czech"},
|
||||
{"25", "Mongolian"},
|
||||
{"26", "Turkish"},
|
||||
{"27", "Indonesian"},
|
||||
{"28", "Korean"},
|
||||
{"29", "Spanish (LATAM)" },
|
||||
{"30", "Persian"},
|
||||
{"31", "Malaysian"}
|
||||
}) { Name = "Language", Value = "1" };
|
||||
configData.AddDynamic("languageid", languageSelect);
|
||||
|
||||
// Configure the sort selects
|
||||
var sortBySelect = new SelectItem(new Dictionary<string, string>()
|
||||
{
|
||||
{"upload_timestamp", "created"},
|
||||
{"seeders", "seeders"},
|
||||
{"size", "size"},
|
||||
{"filename", "title"}
|
||||
}) { Name = "Sort by", Value = "upload_timestamp" };
|
||||
configData.AddDynamic("sortrequestedfromsite", sortBySelect);
|
||||
|
||||
var orderSelect = new SelectItem(new Dictionary<string, string>()
|
||||
{
|
||||
{"desc", "Descending"},
|
||||
{"asc", "Ascending"}
|
||||
})
|
||||
{ Name = "Order", Value = "desc" };
|
||||
configData.AddDynamic("orderrequestedfromsite", orderSelect);
|
||||
}
|
||||
|
||||
private string GetSortBy => ((SelectItem)configData.GetDynamic("sortrequestedfromsite")).Value;
|
||||
|
||||
private string GetOrder => ((SelectItem)configData.GetDynamic("orderrequestedfromsite")).Value;
|
||||
|
||||
private Uri GetAbsoluteUrl(string relativeUrl) => new Uri(SiteLink + relativeUrl.TrimStart('/'));
|
||||
|
||||
public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
|
||||
{
|
||||
LoadValuesFromJson(configJson);
|
||||
var releases = await PerformQuery(new TorznabQuery());
|
||||
|
||||
await ConfigureIfOK(string.Empty, releases.Any(), () =>
|
||||
throw new Exception("Could not find releases from this URL"));
|
||||
|
||||
return IndexerConfigurationStatus.Completed;
|
||||
}
|
||||
|
||||
protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
|
||||
{
|
||||
try
|
||||
{
|
||||
await ConfigureDDoSGuardCookie();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.Log(LogLevel.Warn, ex, $"Exception while configuring DDoS Guard cookie. Attempting search without. (Exception: {ex})");
|
||||
}
|
||||
|
||||
// Get specified categories. If none were specified, use all available.
|
||||
var searchCategories = MapTorznabCapsToTrackers(query);
|
||||
if (searchCategories.Count == 0)
|
||||
searchCategories = GetAllTrackerCategories();
|
||||
|
||||
// Prepare the search query
|
||||
var queryParameters = new NameValueCollection
|
||||
{
|
||||
{ "page", "search" },
|
||||
{ "id", string.Join(",", searchCategories) },
|
||||
{ "group", "0" }, // No group
|
||||
{ "q", query.SearchTerm ?? string.Empty },
|
||||
{ "s", GetSortBy },
|
||||
{ "o", GetOrder }
|
||||
};
|
||||
|
||||
// Make search request
|
||||
var searchUri = GetAbsoluteUrl("?" + queryParameters.GetQueryString());
|
||||
var response = await RequestStringWithCookiesAndRetry(searchUri.AbsoluteUri);
|
||||
|
||||
// Check for DDOS Guard or other error
|
||||
if (response.Status == System.Net.HttpStatusCode.Forbidden)
|
||||
throw new IOException("Anidex search was forbidden. This was likely caused by DDOS protection.");
|
||||
|
||||
if (response.Status != System.Net.HttpStatusCode.OK)
|
||||
throw new IOException($"Anidex search returned unexpected result. Expected 200 OK but got {response.Status.ToString()}.");
|
||||
|
||||
// Search seems to have been a success so parse it
|
||||
return ParseResult(response.Content);
|
||||
}
|
||||
|
||||
private IEnumerable<ReleaseInfo> ParseResult(string response)
|
||||
{
|
||||
const string rowSelector = "div#content table > tbody > tr";
|
||||
|
||||
try
|
||||
{
|
||||
var resultParser = new HtmlParser();
|
||||
var resultDocument = resultParser.ParseDocument(response);
|
||||
IEnumerable<IElement> rows = resultDocument.QuerySelectorAll(rowSelector);
|
||||
|
||||
var releases = new List<ReleaseInfo>();
|
||||
foreach (var r in rows)
|
||||
try
|
||||
{
|
||||
var release = new ReleaseInfo();
|
||||
|
||||
release.Category = ParseValueFromRow(r, nameof(release.Category), "td:nth-child(1) a", (e) => MapTrackerCatToNewznab(e.Attributes["href"].Value.Substring(5)));
|
||||
release.Title = ParseStringValueFromRow(r, nameof(release.Title), "td:nth-child(3) span");
|
||||
release.Link = ParseValueFromRow(r, nameof(release.Link), "a[href^=\"/dl/\"]", (e) => GetAbsoluteUrl(e.Attributes["href"].Value));
|
||||
release.MagnetUri = ParseValueFromRow(r, nameof(release.MagnetUri), "a[href^=\"magnet:?\"]", (e) => new Uri(e.Attributes["href"].Value));
|
||||
release.Size = ParseValueFromRow(r, nameof(release.Size), "td:nth-child(7)", (e) => ReleaseInfo.GetBytes(e.Text()));
|
||||
release.PublishDate = ParseValueFromRow(r, nameof(release.PublishDate), "td:nth-child(8)", (e) => DateTime.ParseExact(e.Attributes["title"].Value, "yyyy-MM-dd HH:mm:ss UTC", CultureInfo.InvariantCulture));
|
||||
release.Seeders = ParseIntValueFromRow(r, nameof(release.Seeders), "td:nth-child(9)");
|
||||
release.Peers = ParseIntValueFromRow(r, nameof(release.Peers), "td:nth-child(10)") + release.Seeders;
|
||||
release.Grabs = ParseIntValueFromRow(r, nameof(release.Grabs), "td:nth-child(11)");
|
||||
release.Comments = ParseValueFromRow(r, nameof(release.Comments), "td:nth-child(3) a", (e) => GetAbsoluteUrl(e.Attributes["href"].Value));
|
||||
release.Guid = release.Comments;
|
||||
release.MinimumRatio = 1;
|
||||
release.MinimumSeedTime = 172800; // 48 hours
|
||||
release.DownloadVolumeFactor = 0;
|
||||
release.UploadVolumeFactor = 1;
|
||||
|
||||
releases.Add(release);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.Error($"Anidex: Error parsing search result row '{r.ToHtmlPretty()}':\n\n{ex}");
|
||||
}
|
||||
|
||||
return releases;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new IOException($"Error parsing search result page: {ex}");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ConfigureDDoSGuardCookie()
|
||||
{
|
||||
const string pathAndQueryBase64Encoded = "Lw=="; // "/"
|
||||
const string baseUriBase64Encoded = "aHR0cHM6Ly9hbmlkZXguaW5mbw=="; // "http://anidex.info"
|
||||
const string ddosPostUrl = "https://ddgu.ddos-guard.net/ddgu/";
|
||||
|
||||
try
|
||||
{
|
||||
// Check if the cookie already exists, if so exit without doing anything
|
||||
if (IsCookiePresent("__ddgu") && IsCookiePresent("__ddg1"))
|
||||
{
|
||||
logger.Debug("DDOS Guard cookies are already present. Skipping bypass.");
|
||||
return;
|
||||
}
|
||||
|
||||
// Make a request to DDoS Guard to get the redirect URL
|
||||
var ddosPostData = new List<KeyValuePair<string, string>>
|
||||
{
|
||||
{ "u", pathAndQueryBase64Encoded },
|
||||
{ "h", baseUriBase64Encoded },
|
||||
{ "p", string.Empty }
|
||||
};
|
||||
|
||||
var result = await PostDataWithCookiesAndRetry(ddosPostUrl, ddosPostData);
|
||||
|
||||
if (!result.IsRedirect)
|
||||
// Success returns a redirect. For anything else, assume a failure.
|
||||
throw new IOException($"Unexpected result from DDOS Guard while attempting to bypass: {result.Content}");
|
||||
|
||||
// Call the redirect URL to retrieve the cookie
|
||||
result = await RequestStringWithCookiesAndRetry(result.RedirectingTo);
|
||||
if (!result.IsRedirect)
|
||||
// Success is another redirect. For anything else, assume a failure.
|
||||
throw new IOException($"Unexpected result when returning from DDOS Guard bypass: {result.Content}");
|
||||
|
||||
// If we got to this point, the bypass should have succeeded and we have stored the necessary cookies to access the site normally.
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new IOException($"Error while configuring DDoS Guard cookie: {ex}");
|
||||
}
|
||||
}
|
||||
|
||||
private bool IsCookiePresent(string name)
|
||||
{
|
||||
var rawCookies = CookieHeader.Split(';');
|
||||
IDictionary<string, string> cookies = rawCookies
|
||||
.Where(e => e.Contains('='))
|
||||
.ToDictionary((e) => e.Split('=')[0].Trim(), (e) => e.Split('=')[1].Trim());
|
||||
|
||||
return cookies.ContainsKey(name);
|
||||
}
|
||||
|
||||
private static TResult ParseValueFromRow<TResult>(IElement row, string propertyName, string selector,
|
||||
Func<IElement, TResult> parseFunction)
|
||||
{
|
||||
try
|
||||
{
|
||||
var selectedElement = row.QuerySelector(selector);
|
||||
if (selectedElement == null)
|
||||
throw new IOException($"Unable to find '{selector}'.");
|
||||
|
||||
return parseFunction(selectedElement);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new IOException($"Error parsing for property '{propertyName}': {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private static string ParseStringValueFromRow(IElement row, string propertyName, string selector)
|
||||
{
|
||||
try
|
||||
{
|
||||
var selectedElement = row.QuerySelector(selector);
|
||||
if (selectedElement == null)
|
||||
throw new IOException($"Unable to find '{selector}'.");
|
||||
|
||||
return selectedElement.Text();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new IOException($"Error parsing for property '{propertyName}': {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private static int ParseIntValueFromRow(IElement row, string propertyName, string selector)
|
||||
{
|
||||
try
|
||||
{
|
||||
var text = ParseStringValueFromRow(row, propertyName, selector);
|
||||
if (!int.TryParse(text, out var value))
|
||||
throw new IOException($"Could not convert '{text}' to int.");
|
||||
return value;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new IOException($"Error parsing for property '{propertyName}': {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user