mirror of
https://github.com/Prowlarr/Prowlarr.git
synced 2025-09-17 17:14:18 +02:00
New: Add TorrentRssIndexer
Co-authored-by: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,92 @@
|
|||||||
|
using System.Collections.Generic;
|
||||||
|
using NLog;
|
||||||
|
using NzbDrone.Core.Configuration;
|
||||||
|
using NzbDrone.Core.Messaging.Events;
|
||||||
|
using NzbDrone.Core.ThingiProvider;
|
||||||
|
|
||||||
|
namespace NzbDrone.Core.Indexers.Definitions.TorrentRss
|
||||||
|
{
|
||||||
|
public class TorrentRssIndexer : TorrentIndexerBase<TorrentRssIndexerSettings>
|
||||||
|
{
|
||||||
|
private readonly ITorrentRssParserFactory _torrentRssParserFactory;
|
||||||
|
|
||||||
|
public override string Name => "Torrent RSS Feed";
|
||||||
|
public override string[] IndexerUrls => new[] { "" };
|
||||||
|
public override string Description => "Generic RSS Feed containing torrents";
|
||||||
|
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
|
||||||
|
public override IndexerPrivacy Privacy => IndexerPrivacy.Public;
|
||||||
|
public override int PageSize => 0;
|
||||||
|
public override IndexerCapabilities Capabilities => SetCapabilities();
|
||||||
|
|
||||||
|
public TorrentRssIndexer(IIndexerHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger, ITorrentRssParserFactory torrentRssParserFactory)
|
||||||
|
: base(httpClient, eventAggregator, indexerStatusService, configService, logger)
|
||||||
|
{
|
||||||
|
_torrentRssParserFactory = torrentRssParserFactory;
|
||||||
|
}
|
||||||
|
|
||||||
|
public override IIndexerRequestGenerator GetRequestGenerator()
|
||||||
|
{
|
||||||
|
return new TorrentRssIndexerRequestGenerator { Settings = Settings };
|
||||||
|
}
|
||||||
|
|
||||||
|
public override IParseIndexerResponse GetParser()
|
||||||
|
{
|
||||||
|
return _torrentRssParserFactory.GetParser(Settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
public override IEnumerable<ProviderDefinition> DefaultDefinitions
|
||||||
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
|
yield return GetDefinition("showRSS", "showRSS is a service that allows you to keep track of your favorite TV shows", GetSettings("https://showrss.info/other/all.rss", allowZeroSize: true, defaultReleaseSize: 512));
|
||||||
|
yield return GetDefinition("Torrent RSS Feed", "Generic RSS Feed containing torrents", GetSettings(""));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private IndexerDefinition GetDefinition(string name, string description, TorrentRssIndexerSettings settings)
|
||||||
|
{
|
||||||
|
return new IndexerDefinition
|
||||||
|
{
|
||||||
|
Enable = true,
|
||||||
|
Name = name,
|
||||||
|
Description = description,
|
||||||
|
Implementation = GetType().Name,
|
||||||
|
Settings = settings,
|
||||||
|
Protocol = DownloadProtocol.Torrent,
|
||||||
|
SupportsRss = SupportsRss,
|
||||||
|
SupportsSearch = SupportsSearch,
|
||||||
|
SupportsRedirect = SupportsRedirect,
|
||||||
|
SupportsPagination = SupportsPagination,
|
||||||
|
Capabilities = Capabilities
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private TorrentRssIndexerSettings GetSettings(string url, bool? allowZeroSize = null, double? defaultReleaseSize = null)
|
||||||
|
{
|
||||||
|
var settings = new TorrentRssIndexerSettings
|
||||||
|
{
|
||||||
|
BaseUrl = url,
|
||||||
|
AllowZeroSize = allowZeroSize.GetValueOrDefault(false)
|
||||||
|
};
|
||||||
|
|
||||||
|
if (defaultReleaseSize.HasValue)
|
||||||
|
{
|
||||||
|
settings.DefaultReleaseSize = defaultReleaseSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
return settings;
|
||||||
|
}
|
||||||
|
|
||||||
|
private IndexerCapabilities SetCapabilities()
|
||||||
|
{
|
||||||
|
var caps = new IndexerCapabilities
|
||||||
|
{
|
||||||
|
SearchParams = new List<SearchParam>(),
|
||||||
|
};
|
||||||
|
|
||||||
|
caps.Categories.AddCategoryMapping(1, NewznabStandardCategory.Other);
|
||||||
|
|
||||||
|
return caps;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,12 @@
|
|||||||
|
namespace NzbDrone.Core.Indexers.Definitions.TorrentRss
|
||||||
|
{
|
||||||
|
public class TorrentRssIndexerParserSettings
|
||||||
|
{
|
||||||
|
public bool UseEZTVFormat { get; set; }
|
||||||
|
public bool ParseSeedersInDescription { get; set; }
|
||||||
|
public bool UseEnclosureUrl { get; set; }
|
||||||
|
public bool UseEnclosureLength { get; set; }
|
||||||
|
public bool ParseSizeInDescription { get; set; }
|
||||||
|
public string SizeElementName { get; set; }
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,63 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using NzbDrone.Common.Extensions;
|
||||||
|
using NzbDrone.Common.Http;
|
||||||
|
using NzbDrone.Core.IndexerSearch.Definitions;
|
||||||
|
|
||||||
|
namespace NzbDrone.Core.Indexers.Definitions.TorrentRss
|
||||||
|
{
|
||||||
|
public class TorrentRssIndexerRequestGenerator : IIndexerRequestGenerator
|
||||||
|
{
|
||||||
|
public TorrentRssIndexerSettings Settings { get; set; }
|
||||||
|
|
||||||
|
public virtual IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
|
||||||
|
{
|
||||||
|
return new IndexerPageableRequestChain();
|
||||||
|
}
|
||||||
|
|
||||||
|
public virtual IndexerPageableRequestChain GetSearchRequests(MusicSearchCriteria searchCriteria)
|
||||||
|
{
|
||||||
|
return new IndexerPageableRequestChain();
|
||||||
|
}
|
||||||
|
|
||||||
|
public virtual IndexerPageableRequestChain GetSearchRequests(TvSearchCriteria searchCriteria)
|
||||||
|
{
|
||||||
|
return new IndexerPageableRequestChain();
|
||||||
|
}
|
||||||
|
|
||||||
|
public virtual IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCriteria)
|
||||||
|
{
|
||||||
|
return new IndexerPageableRequestChain();
|
||||||
|
}
|
||||||
|
|
||||||
|
public virtual IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchCriteria)
|
||||||
|
{
|
||||||
|
var pageableRequests = new IndexerPageableRequestChain();
|
||||||
|
|
||||||
|
if (searchCriteria.IsRssSearch)
|
||||||
|
{
|
||||||
|
pageableRequests.Add(GetRssRequests());
|
||||||
|
}
|
||||||
|
|
||||||
|
return pageableRequests;
|
||||||
|
}
|
||||||
|
|
||||||
|
private IEnumerable<IndexerRequest> GetRssRequests()
|
||||||
|
{
|
||||||
|
var request = new IndexerRequest(Settings.BaseUrl.Trim().TrimEnd('/'), HttpAccept.Rss);
|
||||||
|
|
||||||
|
if (Settings.Cookie.IsNotNullOrWhiteSpace())
|
||||||
|
{
|
||||||
|
foreach (var cookie in HttpHeader.ParseCookies(Settings.Cookie))
|
||||||
|
{
|
||||||
|
request.HttpRequest.Cookies[cookie.Key] = cookie.Value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
yield return request;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Func<IDictionary<string, string>> GetCookies { get; set; }
|
||||||
|
public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,51 @@
|
|||||||
|
using FluentValidation;
|
||||||
|
using NzbDrone.Core.Annotations;
|
||||||
|
using NzbDrone.Core.Validation;
|
||||||
|
|
||||||
|
namespace NzbDrone.Core.Indexers.Definitions.TorrentRss
|
||||||
|
{
|
||||||
|
public class TorrentRssIndexerSettingsValidator : AbstractValidator<TorrentRssIndexerSettings>
|
||||||
|
{
|
||||||
|
public TorrentRssIndexerSettingsValidator()
|
||||||
|
{
|
||||||
|
RuleFor(c => c.BaseUrl).ValidRootUrl();
|
||||||
|
|
||||||
|
RuleFor(c => c.BaseSettings).SetValidator(new IndexerCommonSettingsValidator());
|
||||||
|
RuleFor(c => c.TorrentBaseSettings).SetValidator(new IndexerTorrentSettingsValidator());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public class TorrentRssIndexerSettings : ITorrentIndexerSettings
|
||||||
|
{
|
||||||
|
private static readonly TorrentRssIndexerSettingsValidator Validator = new ();
|
||||||
|
|
||||||
|
public TorrentRssIndexerSettings()
|
||||||
|
{
|
||||||
|
BaseUrl = string.Empty;
|
||||||
|
AllowZeroSize = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
[FieldDefinition(0, Label = "Full RSS Feed URL", HelpTextWarning = "To sync to your apps you will need to include the 8000(Other) in the Sync Categories", HelpLink = "https://wiki.servarr.com/en/prowlarr/faq#can-i-add-any-generic-torrent-rss-feed")]
|
||||||
|
public string BaseUrl { get; set; }
|
||||||
|
|
||||||
|
[FieldDefinition(1, Label = "Cookie", HelpText = "If your site requires a login cookie to access the RSS, you'll have to retrieve it via a browser.")]
|
||||||
|
public string Cookie { get; set; }
|
||||||
|
|
||||||
|
[FieldDefinition(2, Type = FieldType.Checkbox, Label = "Allow Zero Size", HelpText="Enabling this will allow you to use feeds that don't specify release size, but be careful, size related checks will not be performed.")]
|
||||||
|
public bool AllowZeroSize { get; set; }
|
||||||
|
|
||||||
|
[FieldDefinition(3, Type = FieldType.Number, Label = "Default Release Size", HelpText="Add a default size for feeds with missing sizes.", Unit = "MB", Advanced = true)]
|
||||||
|
public double? DefaultReleaseSize { get; set; }
|
||||||
|
|
||||||
|
[FieldDefinition(20)]
|
||||||
|
public IndexerBaseSettings BaseSettings { get; set; } = new ();
|
||||||
|
|
||||||
|
[FieldDefinition(21)]
|
||||||
|
public IndexerTorrentBaseSettings TorrentBaseSettings { get; set; } = new ();
|
||||||
|
|
||||||
|
public NzbDroneValidationResult Validate()
|
||||||
|
{
|
||||||
|
return new NzbDroneValidationResult(Validator.Validate(this));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,65 @@
|
|||||||
|
using System;
|
||||||
|
using NLog;
|
||||||
|
using NzbDrone.Common.Cache;
|
||||||
|
using NzbDrone.Common.Serializer;
|
||||||
|
using NzbDrone.Core.Indexers.Exceptions;
|
||||||
|
|
||||||
|
namespace NzbDrone.Core.Indexers.Definitions.TorrentRss
|
||||||
|
{
|
||||||
|
public interface ITorrentRssParserFactory
|
||||||
|
{
|
||||||
|
TorrentRssParser GetParser(TorrentRssIndexerSettings settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
public class TorrentRssParserFactory : ITorrentRssParserFactory
|
||||||
|
{
|
||||||
|
protected readonly Logger _logger;
|
||||||
|
|
||||||
|
private readonly ICached<TorrentRssIndexerParserSettings> _settingsCache;
|
||||||
|
|
||||||
|
private readonly ITorrentRssSettingsDetector _torrentRssSettingsDetector;
|
||||||
|
|
||||||
|
public TorrentRssParserFactory(ICacheManager cacheManager, ITorrentRssSettingsDetector torrentRssSettingsDetector, Logger logger)
|
||||||
|
{
|
||||||
|
_settingsCache = cacheManager.GetCache<TorrentRssIndexerParserSettings>(GetType());
|
||||||
|
_torrentRssSettingsDetector = torrentRssSettingsDetector;
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public TorrentRssParser GetParser(TorrentRssIndexerSettings indexerSettings)
|
||||||
|
{
|
||||||
|
var key = indexerSettings.ToJson();
|
||||||
|
var parserSettings = _settingsCache.Get(key, () => DetectParserSettings(indexerSettings), TimeSpan.FromDays(7));
|
||||||
|
|
||||||
|
if (parserSettings.UseEZTVFormat)
|
||||||
|
{
|
||||||
|
return new EzrssTorrentRssParser();
|
||||||
|
}
|
||||||
|
|
||||||
|
return new TorrentRssParser
|
||||||
|
{
|
||||||
|
UseGuidInfoUrl = false,
|
||||||
|
ParseSeedersInDescription = parserSettings.ParseSeedersInDescription,
|
||||||
|
|
||||||
|
UseEnclosureUrl = parserSettings.UseEnclosureUrl,
|
||||||
|
UseEnclosureLength = parserSettings.UseEnclosureLength,
|
||||||
|
ParseSizeInDescription = parserSettings.ParseSizeInDescription,
|
||||||
|
SizeElementName = parserSettings.SizeElementName,
|
||||||
|
|
||||||
|
DefaultReleaseSize = indexerSettings.DefaultReleaseSize
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private TorrentRssIndexerParserSettings DetectParserSettings(TorrentRssIndexerSettings indexerSettings)
|
||||||
|
{
|
||||||
|
var settings = _torrentRssSettingsDetector.Detect(indexerSettings);
|
||||||
|
|
||||||
|
if (settings == null)
|
||||||
|
{
|
||||||
|
throw new UnsupportedFeedException("Could not parse feed from {0}", indexerSettings.BaseUrl);
|
||||||
|
}
|
||||||
|
|
||||||
|
return settings;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@@ -0,0 +1,305 @@
|
|||||||
|
using System;
|
||||||
|
using System.IO;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Xml;
|
||||||
|
using System.Xml.Linq;
|
||||||
|
using NLog;
|
||||||
|
using NzbDrone.Common.Extensions;
|
||||||
|
using NzbDrone.Common.Http;
|
||||||
|
using NzbDrone.Core.Indexers.Exceptions;
|
||||||
|
using NzbDrone.Core.IndexerSearch.Definitions;
|
||||||
|
using NzbDrone.Core.Parser.Model;
|
||||||
|
|
||||||
|
namespace NzbDrone.Core.Indexers.Definitions.TorrentRss
|
||||||
|
{
|
||||||
|
public interface ITorrentRssSettingsDetector
|
||||||
|
{
|
||||||
|
TorrentRssIndexerParserSettings Detect(TorrentRssIndexerSettings settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
public class TorrentRssSettingsDetector : ITorrentRssSettingsDetector
|
||||||
|
{
|
||||||
|
private const long ValidSizeThreshold = 2 * 1024 * 1024;
|
||||||
|
|
||||||
|
private readonly IHttpClient _httpClient;
|
||||||
|
private readonly Logger _logger;
|
||||||
|
|
||||||
|
public TorrentRssSettingsDetector(IHttpClient httpClient, Logger logger)
|
||||||
|
{
|
||||||
|
_httpClient = httpClient;
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public TorrentRssIndexerParserSettings Detect(TorrentRssIndexerSettings settings)
|
||||||
|
{
|
||||||
|
_logger.Debug("Evaluating TorrentRss feed '{0}'", settings.BaseUrl);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var requestGenerator = new TorrentRssIndexerRequestGenerator { Settings = settings };
|
||||||
|
var request = requestGenerator.GetSearchRequests(new BasicSearchCriteria()).GetAllTiers().First().First();
|
||||||
|
|
||||||
|
HttpResponse httpResponse;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
httpResponse = _httpClient.Execute(request.HttpRequest);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.Warn(ex, "Unable to connect to indexer {0}: {1}", request.Url, ex.Message);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var indexerResponse = new IndexerResponse(request, httpResponse);
|
||||||
|
|
||||||
|
return GetParserSettings(indexerResponse, settings);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
ex.WithData("FeedUrl", settings.BaseUrl);
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private TorrentRssIndexerParserSettings GetParserSettings(IndexerResponse response, TorrentRssIndexerSettings indexerSettings)
|
||||||
|
{
|
||||||
|
var settings = GetEzrssParserSettings(response, indexerSettings);
|
||||||
|
|
||||||
|
if (settings != null)
|
||||||
|
{
|
||||||
|
return settings;
|
||||||
|
}
|
||||||
|
|
||||||
|
settings = GetGenericTorrentRssParserSettings(response, indexerSettings);
|
||||||
|
|
||||||
|
if (settings != null)
|
||||||
|
{
|
||||||
|
return settings;
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private TorrentRssIndexerParserSettings GetEzrssParserSettings(IndexerResponse response, TorrentRssIndexerSettings indexerSettings)
|
||||||
|
{
|
||||||
|
if (!IsEZTVFeed(response))
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.Trace("Feed has Ezrss schema");
|
||||||
|
|
||||||
|
var parser = new EzrssTorrentRssParser();
|
||||||
|
var releases = ParseResponse(parser, response);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
ValidateReleases(releases, indexerSettings);
|
||||||
|
ValidateReleaseSize(releases, indexerSettings);
|
||||||
|
|
||||||
|
_logger.Debug("Feed was parseable by Ezrss Parser");
|
||||||
|
return new TorrentRssIndexerParserSettings
|
||||||
|
{
|
||||||
|
UseEZTVFormat = true
|
||||||
|
};
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.Trace(ex, "Feed wasn't parsable by Ezrss Parser");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private TorrentRssIndexerParserSettings GetGenericTorrentRssParserSettings(IndexerResponse response, TorrentRssIndexerSettings indexerSettings)
|
||||||
|
{
|
||||||
|
var parser = new TorrentRssParser
|
||||||
|
{
|
||||||
|
UseEnclosureUrl = true,
|
||||||
|
UseEnclosureLength = true,
|
||||||
|
ParseSeedersInDescription = true
|
||||||
|
};
|
||||||
|
|
||||||
|
var item = parser.GetItems(response).FirstOrDefault();
|
||||||
|
if (item == null)
|
||||||
|
{
|
||||||
|
throw new UnsupportedFeedException("Empty feed, cannot check if feed is parsable.");
|
||||||
|
}
|
||||||
|
|
||||||
|
var settings = new TorrentRssIndexerParserSettings()
|
||||||
|
{
|
||||||
|
UseEnclosureUrl = true,
|
||||||
|
UseEnclosureLength = true,
|
||||||
|
ParseSeedersInDescription = true
|
||||||
|
};
|
||||||
|
|
||||||
|
if (item.Element("enclosure") == null)
|
||||||
|
{
|
||||||
|
parser.UseEnclosureUrl = settings.UseEnclosureUrl = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
var releases = ParseResponse(parser, response);
|
||||||
|
ValidateReleases(releases, indexerSettings);
|
||||||
|
|
||||||
|
if (!releases.Any(v => v.Seeders.HasValue))
|
||||||
|
{
|
||||||
|
_logger.Trace("Feed doesn't have Seeders in Description, disabling option.");
|
||||||
|
parser.ParseSeedersInDescription = settings.ParseSeedersInDescription = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!releases.Any(r => r.Size < ValidSizeThreshold))
|
||||||
|
{
|
||||||
|
_logger.Trace("Feed has valid size in enclosure.");
|
||||||
|
return settings;
|
||||||
|
}
|
||||||
|
|
||||||
|
parser.UseEnclosureLength = settings.UseEnclosureLength = false;
|
||||||
|
|
||||||
|
foreach (var sizeElementName in new[] { "size", "Size" })
|
||||||
|
{
|
||||||
|
parser.SizeElementName = settings.SizeElementName = sizeElementName;
|
||||||
|
|
||||||
|
releases = ParseResponse(parser, response);
|
||||||
|
ValidateReleases(releases, indexerSettings);
|
||||||
|
|
||||||
|
if (!releases.Any(r => r.Size < ValidSizeThreshold))
|
||||||
|
{
|
||||||
|
_logger.Trace("Feed has valid size in Size element.");
|
||||||
|
return settings;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
parser.SizeElementName = settings.SizeElementName = null;
|
||||||
|
parser.ParseSizeInDescription = settings.ParseSizeInDescription = true;
|
||||||
|
|
||||||
|
releases = ParseResponse(parser, response);
|
||||||
|
ValidateReleases(releases, indexerSettings);
|
||||||
|
|
||||||
|
if (releases.Count(r => r.Size >= ValidSizeThreshold) > releases.Length / 2)
|
||||||
|
{
|
||||||
|
if (releases.Any(r => r.Size < ValidSizeThreshold))
|
||||||
|
{
|
||||||
|
_logger.Debug("Feed {0} contains very small releases.", response.Request.Url);
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.Trace("Feed has valid size in description.");
|
||||||
|
return settings;
|
||||||
|
}
|
||||||
|
|
||||||
|
parser.ParseSizeInDescription = settings.ParseSizeInDescription = false;
|
||||||
|
|
||||||
|
_logger.Debug("Feed doesn't have release size.");
|
||||||
|
|
||||||
|
releases = ParseResponse(parser, response);
|
||||||
|
ValidateReleases(releases, indexerSettings);
|
||||||
|
ValidateReleaseSize(releases, indexerSettings);
|
||||||
|
|
||||||
|
return settings;
|
||||||
|
}
|
||||||
|
|
||||||
|
private bool IsEZTVFeed(IndexerResponse response)
|
||||||
|
{
|
||||||
|
var content = XmlCleaner.ReplaceEntities(response.Content);
|
||||||
|
content = XmlCleaner.ReplaceUnicode(content);
|
||||||
|
|
||||||
|
using var xmlTextReader = XmlReader.Create(new StringReader(content), new XmlReaderSettings { DtdProcessing = DtdProcessing.Parse, ValidationType = ValidationType.None, IgnoreComments = true, XmlResolver = null });
|
||||||
|
var document = XDocument.Load(xmlTextReader);
|
||||||
|
|
||||||
|
// Check Namespace
|
||||||
|
if (document.Root == null)
|
||||||
|
{
|
||||||
|
throw new InvalidDataException("Could not parse IndexerResponse into XML.");
|
||||||
|
}
|
||||||
|
|
||||||
|
var ns = document.Root.GetNamespaceOfPrefix("torrent");
|
||||||
|
if (ns == "http://xmlns.ezrss.it/0.1/")
|
||||||
|
{
|
||||||
|
_logger.Trace("Identified feed as EZTV compatible by EZTV Namespace");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check DTD in DocType
|
||||||
|
if (document.DocumentType != null && document.DocumentType.SystemId == "http://xmlns.ezrss.it/0.1/dtd/")
|
||||||
|
{
|
||||||
|
_logger.Trace("Identified feed as EZTV compatible by EZTV DTD");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check namespaces
|
||||||
|
if (document.Descendants().Any(v => v.GetDefaultNamespace().NamespaceName == "http://xmlns.ezrss.it/0.1/"))
|
||||||
|
{
|
||||||
|
_logger.Trace("Identified feed as EZTV compatible by EZTV Namespace");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
private TorrentInfo[] ParseResponse(IParseIndexerResponse parser, IndexerResponse response)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var releases = parser.ParseResponse(response).Cast<TorrentInfo>().ToArray();
|
||||||
|
return releases;
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.Debug(ex, "Unable to parse indexer feed: " + ex.Message);
|
||||||
|
throw new UnsupportedFeedException("Unable to parse indexer: " + ex.Message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ValidateReleases(TorrentInfo[] releases, TorrentRssIndexerSettings indexerSettings)
|
||||||
|
{
|
||||||
|
if (releases == null || releases.Empty())
|
||||||
|
{
|
||||||
|
throw new UnsupportedFeedException("Empty feed, cannot check if feed is parsable.");
|
||||||
|
}
|
||||||
|
|
||||||
|
var torrentInfo = releases.First();
|
||||||
|
|
||||||
|
_logger.Trace("TorrentInfo: \n{0}", torrentInfo.ToString("L"));
|
||||||
|
|
||||||
|
if (releases.Any(r => r.Title.IsNullOrWhiteSpace()))
|
||||||
|
{
|
||||||
|
throw new UnsupportedFeedException("Feed contains releases without title.");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (releases.Any(r => !IsValidDownloadUrl(r.DownloadUrl)))
|
||||||
|
{
|
||||||
|
throw new UnsupportedFeedException("Failed to find a valid download url in the feed.");
|
||||||
|
}
|
||||||
|
|
||||||
|
var total = releases.Where(v => v.Guid != null).Select(v => v.Guid).ToArray();
|
||||||
|
var distinct = total.Distinct().ToArray();
|
||||||
|
|
||||||
|
if (distinct.Length != total.Length)
|
||||||
|
{
|
||||||
|
throw new UnsupportedFeedException("Feed contains releases with same guid, rejecting malformed rss feed.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ValidateReleaseSize(TorrentInfo[] releases, TorrentRssIndexerSettings indexerSettings)
|
||||||
|
{
|
||||||
|
if (!indexerSettings.AllowZeroSize && releases.Any(r => r.Size == 0))
|
||||||
|
{
|
||||||
|
throw new UnsupportedFeedException("Feed doesn't contain the release content size.");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (releases.Any(r => r.Size != 0 && r.Size < ValidSizeThreshold))
|
||||||
|
{
|
||||||
|
throw new UnsupportedFeedException("Size of one or more releases lower than {0}, feed must contain release content size.", ValidSizeThreshold.SizeSuffix());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsValidDownloadUrl(string url)
|
||||||
|
{
|
||||||
|
if (url.IsNullOrWhiteSpace())
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return url.StartsWith("magnet:") || url.StartsWith("http:") || url.StartsWith("https:");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@@ -177,9 +177,10 @@ namespace NzbDrone.Core.Indexers
|
|||||||
}
|
}
|
||||||
|
|
||||||
var definitions = provider.DefaultDefinitions
|
var definitions = provider.DefaultDefinitions
|
||||||
.Where(v => v.Name != null && v.Name != nameof(Cardigann) && v.Name != nameof(Newznab.Newznab) && v.Name != nameof(Torznab.Torznab));
|
.Where(v => v.Name != null && v.Name != nameof(Cardigann) && v.Name != nameof(Newznab.Newznab) && v.Name != nameof(Torznab.Torznab))
|
||||||
|
.Cast<IndexerDefinition>();
|
||||||
|
|
||||||
foreach (IndexerDefinition definition in definitions)
|
foreach (var definition in definitions)
|
||||||
{
|
{
|
||||||
SetProviderCharacteristics(provider, definition);
|
SetProviderCharacteristics(provider, definition);
|
||||||
yield return definition;
|
yield return definition;
|
||||||
|
@@ -206,7 +206,7 @@ namespace NzbDrone.Core.Indexers
|
|||||||
|
|
||||||
if (dateString.IsNullOrWhiteSpace())
|
if (dateString.IsNullOrWhiteSpace())
|
||||||
{
|
{
|
||||||
throw new UnsupportedFeedException("Rss feed must have a pubDate element with a valid publish date.");
|
throw new UnsupportedFeedException("Each item in the RSS feed must have a pubDate element with a valid publish date.");
|
||||||
}
|
}
|
||||||
|
|
||||||
return XElementExtensions.ParseDate(dateString);
|
return XElementExtensions.ParseDate(dateString);
|
||||||
@@ -279,9 +279,9 @@ namespace NzbDrone.Core.Indexers
|
|||||||
{
|
{
|
||||||
return new RssEnclosure
|
return new RssEnclosure
|
||||||
{
|
{
|
||||||
Url = v.Attribute("url").Value,
|
Url = v.Attribute("url")?.Value,
|
||||||
Type = v.Attribute("type").Value,
|
Type = v.Attribute("type")?.Value,
|
||||||
Length = (long)v.Attribute("length")
|
Length = v.Attribute("length")?.Value?.ParseInt64() ?? 0
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
catch (Exception e)
|
catch (Exception e)
|
||||||
|
@@ -1,4 +1,5 @@
|
|||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
using System.Text.RegularExpressions;
|
using System.Text.RegularExpressions;
|
||||||
using System.Xml.Linq;
|
using System.Xml.Linq;
|
||||||
using MonoTorrent;
|
using MonoTorrent;
|
||||||
@@ -9,12 +10,30 @@ namespace NzbDrone.Core.Indexers
|
|||||||
{
|
{
|
||||||
public class TorrentRssParser : RssParser
|
public class TorrentRssParser : RssParser
|
||||||
{
|
{
|
||||||
|
// Use to sum/calculate Peers as Leechers+Seeders
|
||||||
|
public bool CalculatePeersAsSum { get; set; }
|
||||||
|
|
||||||
|
// Use the specified element name to determine the Infohash
|
||||||
|
public string InfoHashElementName { get; set; }
|
||||||
|
|
||||||
// Parse various seeder/leecher/peers formats in the description element to determine number of seeders.
|
// Parse various seeder/leecher/peers formats in the description element to determine number of seeders.
|
||||||
public bool ParseSeedersInDescription { get; set; }
|
public bool ParseSeedersInDescription { get; set; }
|
||||||
|
|
||||||
// Use the specified element name to determine the size
|
// Use the specified element name to determine the Peers
|
||||||
|
public string PeersElementName { get; set; }
|
||||||
|
|
||||||
|
// Use the specified element name to determine the Seeds
|
||||||
|
public string SeedsElementName { get; set; }
|
||||||
|
|
||||||
|
// Use the specified element name to determine the Size
|
||||||
public string SizeElementName { get; set; }
|
public string SizeElementName { get; set; }
|
||||||
|
|
||||||
|
// Use the specified element name to determine the Magnet link
|
||||||
|
public string MagnetElementName { get; set; }
|
||||||
|
|
||||||
|
// Default size for when release sizes aren't available
|
||||||
|
public double? DefaultReleaseSize { get; set; }
|
||||||
|
|
||||||
public TorrentRssParser()
|
public TorrentRssParser()
|
||||||
{
|
{
|
||||||
PreferredEnclosureMimeTypes = TorrentEnclosureMimeTypes;
|
PreferredEnclosureMimeTypes = TorrentEnclosureMimeTypes;
|
||||||
@@ -40,14 +59,28 @@ namespace NzbDrone.Core.Indexers
|
|||||||
result.InfoHash = GetInfoHash(item);
|
result.InfoHash = GetInfoHash(item);
|
||||||
result.MagnetUrl = GetMagnetUrl(item);
|
result.MagnetUrl = GetMagnetUrl(item);
|
||||||
result.Seeders = GetSeeders(item);
|
result.Seeders = GetSeeders(item);
|
||||||
|
|
||||||
|
if (CalculatePeersAsSum)
|
||||||
|
{
|
||||||
|
result.Peers = GetPeers(item) + result.Seeders;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
result.Peers = GetPeers(item);
|
result.Peers = GetPeers(item);
|
||||||
|
}
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected virtual string GetInfoHash(XElement item)
|
protected virtual string GetInfoHash(XElement item)
|
||||||
{
|
{
|
||||||
|
if (InfoHashElementName.IsNotNullOrWhiteSpace())
|
||||||
|
{
|
||||||
|
return item.FindDecendants(InfoHashElementName).FirstOrDefault().Value;
|
||||||
|
}
|
||||||
|
|
||||||
var magnetUrl = GetMagnetUrl(item);
|
var magnetUrl = GetMagnetUrl(item);
|
||||||
|
|
||||||
if (magnetUrl.IsNotNullOrWhiteSpace())
|
if (magnetUrl.IsNotNullOrWhiteSpace())
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
@@ -63,18 +96,31 @@ namespace NzbDrone.Core.Indexers
|
|||||||
}
|
}
|
||||||
|
|
||||||
protected virtual string GetMagnetUrl(XElement item)
|
protected virtual string GetMagnetUrl(XElement item)
|
||||||
|
{
|
||||||
|
if (MagnetElementName.IsNotNullOrWhiteSpace())
|
||||||
|
{
|
||||||
|
var magnetURL = item.FindDecendants(MagnetElementName).FirstOrDefault().Value;
|
||||||
|
if (magnetURL.IsNotNullOrWhiteSpace() && magnetURL.StartsWith("magnet:"))
|
||||||
|
{
|
||||||
|
return magnetURL;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
{
|
{
|
||||||
var downloadUrl = GetDownloadUrl(item);
|
var downloadUrl = GetDownloadUrl(item);
|
||||||
if (downloadUrl.IsNotNullOrWhiteSpace() && downloadUrl.StartsWith("magnet:"))
|
if (downloadUrl.IsNotNullOrWhiteSpace() && downloadUrl.StartsWith("magnet:"))
|
||||||
{
|
{
|
||||||
return downloadUrl;
|
return downloadUrl;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected virtual int? GetSeeders(XElement item)
|
protected virtual int? GetSeeders(XElement item)
|
||||||
{
|
{
|
||||||
|
// safe to always use the element if it's present (and valid)
|
||||||
|
// fall back to description if ParseSeedersInDescription is enabled
|
||||||
if (ParseSeedersInDescription && item.Element("description") != null)
|
if (ParseSeedersInDescription && item.Element("description") != null)
|
||||||
{
|
{
|
||||||
var matchSeeders = ParseSeedersRegex.Match(item.Element("description").Value);
|
var matchSeeders = ParseSeedersRegex.Match(item.Element("description").Value);
|
||||||
@@ -93,6 +139,12 @@ namespace NzbDrone.Core.Indexers
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var seeds = item.FindDecendants(SeedsElementName).SingleOrDefault();
|
||||||
|
if (seeds != null)
|
||||||
|
{
|
||||||
|
return (int)seeds;
|
||||||
|
}
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -116,6 +168,12 @@ namespace NzbDrone.Core.Indexers
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (PeersElementName.IsNotNullOrWhiteSpace())
|
||||||
|
{
|
||||||
|
var itempeers = item.FindDecendants(PeersElementName).SingleOrDefault();
|
||||||
|
return int.Parse(itempeers.Value);
|
||||||
|
}
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -124,12 +182,18 @@ namespace NzbDrone.Core.Indexers
|
|||||||
var size = base.GetSize(item);
|
var size = base.GetSize(item);
|
||||||
if (size == 0 && SizeElementName.IsNotNullOrWhiteSpace())
|
if (size == 0 && SizeElementName.IsNotNullOrWhiteSpace())
|
||||||
{
|
{
|
||||||
if (item.Element(SizeElementName) != null)
|
var itemsize = item.FindDecendants(SizeElementName).SingleOrDefault();
|
||||||
|
if (itemsize != null)
|
||||||
{
|
{
|
||||||
size = ParseSize(item.Element(SizeElementName).Value, true);
|
size = ParseSize(itemsize.Value, true);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (size == 0 && DefaultReleaseSize is > 0)
|
||||||
|
{
|
||||||
|
return (long)(DefaultReleaseSize * 1024f * 1024f);
|
||||||
|
}
|
||||||
|
|
||||||
return size;
|
return size;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Reference in New Issue
Block a user