shpaq/Prowlarr-Prowlarr
1
0
Fork 0
mirror of https://github.com/Prowlarr/Prowlarr.git synced 2025-09-17 17:14:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fixed: Push Downloads to client fails for download overrides

Browse Source
This commit is contained in:
Qstick
2021-05-31 21:51:43 -04:00
parent cf1c44ed75
commit 85be0be455
55 changed files with 702 additions and 236 deletions
Download Patch File Download Diff File Expand all files Collapse all files

227
src/NzbDrone.Core.Test/Download/DownloadClientProviderFixture.cs Normal file
View File

@@ -0,0 +1,227 @@
using System;
using System.Collections.Generic;
using FizzWare.NBuilder;
using FluentAssertions;
using Moq;
using NUnit.Framework;
using NzbDrone.Core.Download;
using NzbDrone.Core.Indexers;
using NzbDrone.Core.Test.Framework;
namespace NzbDrone.Core.Test.Download
{
[TestFixture]
public class DownloadClientProviderFixture : CoreTest<DownloadClientProvider>
{
private List<IDownloadClient> _downloadClients;
private List<DownloadClientStatus> _blockedProviders;
private int _nextId;
[SetUp]
public void SetUp()
{
_downloadClients = new List<IDownloadClient>();
_blockedProviders = new List<DownloadClientStatus>();
_nextId = 1;
Mocker.GetMock<IDownloadClientFactory>()
.Setup(v => v.GetAvailableProviders())
.Returns(_downloadClients);
Mocker.GetMock<IDownloadClientStatusService>()
.Setup(v => v.GetBlockedProviders())
.Returns(_blockedProviders);
}
private Mock<IDownloadClient> WithUsenetClient(int priority = 0)
{
var mock = new Mock<IDownloadClient>(MockBehavior.Default);
mock.SetupGet(s => s.Definition)
.Returns(Builder<DownloadClientDefinition>
.CreateNew()
.With(v => v.Id = _nextId++)
.With(v => v.Priority = priority)
.Build());
_downloadClients.Add(mock.Object);
mock.SetupGet(v => v.Protocol).Returns(DownloadProtocol.Usenet);
return mock;
}
private Mock<IDownloadClient> WithTorrentClient(int priority = 0)
{
var mock = new Mock<IDownloadClient>(MockBehavior.Default);
mock.SetupGet(s => s.Definition)
.Returns(Builder<DownloadClientDefinition>
.CreateNew()
.With(v => v.Id = _nextId++)
.With(v => v.Priority = priority)
.Build());
_downloadClients.Add(mock.Object);
mock.SetupGet(v => v.Protocol).Returns(DownloadProtocol.Torrent);
return mock;
}
private void GivenBlockedClient(int id)
{
_blockedProviders.Add(new DownloadClientStatus
{
ProviderId = id,
DisabledTill = DateTime.UtcNow.AddHours(3)
});
}
[Test]
public void should_roundrobin_over_usenet_client()
{
WithUsenetClient();
WithUsenetClient();
WithUsenetClient();
WithTorrentClient();
var client1 = Subject.GetDownloadClient(DownloadProtocol.Usenet);
var client2 = Subject.GetDownloadClient(DownloadProtocol.Usenet);
var client3 = Subject.GetDownloadClient(DownloadProtocol.Usenet);
var client4 = Subject.GetDownloadClient(DownloadProtocol.Usenet);
var client5 = Subject.GetDownloadClient(DownloadProtocol.Usenet);
client1.Definition.Id.Should().Be(1);
client2.Definition.Id.Should().Be(2);
client3.Definition.Id.Should().Be(3);
client4.Definition.Id.Should().Be(1);
client5.Definition.Id.Should().Be(2);
}
[Test]
public void should_roundrobin_over_torrent_client()
{
WithUsenetClient();
WithTorrentClient();
WithTorrentClient();
WithTorrentClient();
var client1 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
var client2 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
var client3 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
var client4 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
var client5 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
client1.Definition.Id.Should().Be(2);
client2.Definition.Id.Should().Be(3);
client3.Definition.Id.Should().Be(4);
client4.Definition.Id.Should().Be(2);
client5.Definition.Id.Should().Be(3);
}
[Test]
public void should_roundrobin_over_protocol_separately()
{
WithUsenetClient();
WithTorrentClient();
WithTorrentClient();
var client1 = Subject.GetDownloadClient(DownloadProtocol.Usenet);
var client2 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
var client3 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
var client4 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
client1.Definition.Id.Should().Be(1);
client2.Definition.Id.Should().Be(2);
client3.Definition.Id.Should().Be(3);
client4.Definition.Id.Should().Be(2);
}
[Test]
public void should_skip_blocked_torrent_client()
{
WithUsenetClient();
WithTorrentClient();
WithTorrentClient();
WithTorrentClient();
GivenBlockedClient(3);
var client1 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
var client2 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
var client3 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
var client4 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
var client5 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
client1.Definition.Id.Should().Be(2);
client2.Definition.Id.Should().Be(4);
client3.Definition.Id.Should().Be(2);
client4.Definition.Id.Should().Be(4);
}
[Test]
public void should_not_skip_blocked_torrent_client_if_all_blocked()
{
WithUsenetClient();
WithTorrentClient();
WithTorrentClient();
WithTorrentClient();
GivenBlockedClient(2);
GivenBlockedClient(3);
GivenBlockedClient(4);
var client1 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
var client2 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
var client3 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
var client4 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
var client5 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
client1.Definition.Id.Should().Be(2);
client2.Definition.Id.Should().Be(3);
client3.Definition.Id.Should().Be(4);
client4.Definition.Id.Should().Be(2);
}
[Test]
public void should_skip_secondary_prio_torrent_client()
{
WithUsenetClient();
WithTorrentClient(2);
WithTorrentClient();
WithTorrentClient();
var client1 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
var client2 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
var client3 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
var client4 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
var client5 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
client1.Definition.Id.Should().Be(3);
client2.Definition.Id.Should().Be(4);
client3.Definition.Id.Should().Be(3);
client4.Definition.Id.Should().Be(4);
}
[Test]
public void should_not_skip_secondary_prio_torrent_client_if_primary_blocked()
{
WithUsenetClient();
WithTorrentClient(2);
WithTorrentClient(2);
WithTorrentClient();
GivenBlockedClient(4);
var client1 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
var client2 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
var client3 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
var client4 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
var client5 = Subject.GetDownloadClient(DownloadProtocol.Torrent);
client1.Definition.Id.Should().Be(2);
client2.Definition.Id.Should().Be(3);
client3.Definition.Id.Should().Be(2);
client4.Definition.Id.Should().Be(3);
}
}
}

161
src/NzbDrone.Core.Test/Download/DownloadClientStatusServiceFixture.cs Normal file
View File

@@ -0,0 +1,161 @@
using System;
using System.Linq;
using FluentAssertions;
using Moq;
using NUnit.Framework;
using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Core.Download;
using NzbDrone.Core.Test.Framework;
namespace NzbDrone.Core.Test.Download
{
public class DownloadClientStatusServiceFixture : CoreTest<DownloadClientStatusService>
{
private DateTime _epoch;
[SetUp]
public void SetUp()
{
_epoch = DateTime.UtcNow;
Mocker.GetMock<IRuntimeInfo>()
.SetupGet(v => v.StartTime)
.Returns(_epoch - TimeSpan.FromHours(1));
}
private DownloadClientStatus WithStatus(DownloadClientStatus status)
{
Mocker.GetMock<IDownloadClientStatusRepository>()
.Setup(v => v.FindByProviderId(1))
.Returns(status);
Mocker.GetMock<IDownloadClientStatusRepository>()
.Setup(v => v.All())
.Returns(new[] { status });
return status;
}
private void VerifyUpdate()
{
Mocker.GetMock<IDownloadClientStatusRepository>()
.Verify(v => v.Upsert(It.IsAny<DownloadClientStatus>()), Times.Once());
}
private void VerifyNoUpdate()
{
Mocker.GetMock<IDownloadClientStatusRepository>()
.Verify(v => v.Upsert(It.IsAny<DownloadClientStatus>()), Times.Never());
}
[Test]
public void should_not_consider_blocked_within_5_minutes_since_initial_failure()
{
WithStatus(new DownloadClientStatus
{
InitialFailure = _epoch - TimeSpan.FromMinutes(4),
MostRecentFailure = _epoch - TimeSpan.FromSeconds(4),
EscalationLevel = 3
});
Subject.RecordFailure(1);
VerifyUpdate();
var status = Subject.GetBlockedProviders().FirstOrDefault();
status.Should().BeNull();
}
[Test]
public void should_consider_blocked_after_5_minutes_since_initial_failure()
{
WithStatus(new DownloadClientStatus
{
InitialFailure = _epoch - TimeSpan.FromMinutes(6),
MostRecentFailure = _epoch - TimeSpan.FromSeconds(120),
EscalationLevel = 3
});
Subject.RecordFailure(1);
VerifyUpdate();
var status = Subject.GetBlockedProviders().FirstOrDefault();
status.Should().NotBeNull();
}
[Test]
public void should_not_escalate_further_till_after_5_minutes_since_initial_failure()
{
var origStatus = WithStatus(new DownloadClientStatus
{
InitialFailure = _epoch - TimeSpan.FromMinutes(4),
MostRecentFailure = _epoch - TimeSpan.FromSeconds(4),
EscalationLevel = 3
});
Subject.RecordFailure(1);
Subject.RecordFailure(1);
Subject.RecordFailure(1);
Subject.RecordFailure(1);
Subject.RecordFailure(1);
Subject.RecordFailure(1);
Subject.RecordFailure(1);
var status = Subject.GetBlockedProviders().FirstOrDefault();
status.Should().BeNull();
origStatus.EscalationLevel.Should().Be(3);
}
[Test]
public void should_escalate_further_after_5_minutes_since_initial_failure()
{
WithStatus(new DownloadClientStatus
{
InitialFailure = _epoch - TimeSpan.FromMinutes(6),
MostRecentFailure = _epoch - TimeSpan.FromSeconds(120),
EscalationLevel = 3
});
Subject.RecordFailure(1);
Subject.RecordFailure(1);
Subject.RecordFailure(1);
Subject.RecordFailure(1);
Subject.RecordFailure(1);
Subject.RecordFailure(1);
Subject.RecordFailure(1);
var status = Subject.GetBlockedProviders().FirstOrDefault();
status.Should().NotBeNull();
status.EscalationLevel.Should().BeGreaterThan(3);
}
[Test]
public void should_not_escalate_beyond_3_hours()
{
WithStatus(new DownloadClientStatus
{
InitialFailure = _epoch - TimeSpan.FromMinutes(6),
MostRecentFailure = _epoch - TimeSpan.FromSeconds(120),
EscalationLevel = 3
});
Subject.RecordFailure(1);
Subject.RecordFailure(1);
Subject.RecordFailure(1);
Subject.RecordFailure(1);
Subject.RecordFailure(1);
Subject.RecordFailure(1);
Subject.RecordFailure(1);
Subject.RecordFailure(1);
Subject.RecordFailure(1);
var status = Subject.GetBlockedProviders().FirstOrDefault();
status.Should().NotBeNull();
status.DisabledTill.Should().HaveValue();
status.DisabledTill.Should().NotBeAfter(_epoch + TimeSpan.FromHours(3.1));
}
}
}

7
src/NzbDrone.Core.Test/IndexerTests/TestIndexer.cs
View File

@@ -1,12 +1,13 @@
using NLog;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Download;
using NzbDrone.Core.Indexers;
using NzbDrone.Core.Messaging.Events;
namespace NzbDrone.Core.Test.IndexerTests
{
public class TestIndexer : HttpIndexerBase<TestIndexerSettings>
public class TestIndexer : UsenetIndexerBase<TestIndexerSettings>
{
public override string Name => "Test Indexer";
public override string BaseUrl => "http://testindexer.com";
@@ -18,8 +19,8 @@ namespace NzbDrone.Core.Test.IndexerTests
public int _supportedPageSize;
public override int PageSize => _supportedPageSize;
public TestIndexer(IHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
: base(httpClient, eventAggregator, indexerStatusService, configService, logger)
public TestIndexer(IHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, IValidateNzbs nzbValidationService, Logger logger)
: base(httpClient, eventAggregator, indexerStatusService, configService, nzbValidationService, logger)
{
}

3
src/NzbDrone.Core/Download/Clients/DownloadStation/UsenetDownloadStation.cs
View File

@@ -31,9 +31,8 @@ namespace NzbDrone.Core.Download.Clients.DownloadStation
IHttpClient httpClient,
IConfigService configService,
IDiskProvider diskProvider,
IValidateNzbs nzbValidationService,
Logger logger)
: base(httpClient, configService, diskProvider, nzbValidationService, logger)
: base(httpClient, configService, diskProvider, logger)
{
_dsInfoProxy = dsInfoProxy;
_dsTaskProxy = dsTaskProxy;

3
src/NzbDrone.Core/Download/Clients/NzbVortex/NzbVortex.cs
View File

@@ -20,9 +20,8 @@ namespace NzbDrone.Core.Download.Clients.NzbVortex
IHttpClient httpClient,
IConfigService configService,
IDiskProvider diskProvider,
IValidateNzbs nzbValidationService,
Logger logger)
: base(httpClient, configService, diskProvider, nzbValidationService, logger)
: base(httpClient, configService, diskProvider, logger)
{
_proxy = proxy;
}

3
src/NzbDrone.Core/Download/Clients/Nzbget/Nzbget.cs
View File

@@ -25,9 +25,8 @@ namespace NzbDrone.Core.Download.Clients.Nzbget
IHttpClient httpClient,
IConfigService configService,
IDiskProvider diskProvider,
IValidateNzbs nzbValidationService,
Logger logger)
: base(httpClient, configService, diskProvider, nzbValidationService, logger)
: base(httpClient, configService, diskProvider, logger)
{
_proxy = proxy;
}

17
src/NzbDrone.Core/Download/Clients/Pneumatic/Pneumatic.cs
View File

@@ -1,5 +1,7 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using FluentValidation.Results;
using NLog;
using NzbDrone.Common.Disk;
@@ -14,24 +16,20 @@ namespace NzbDrone.Core.Download.Clients.Pneumatic
{
public class Pneumatic : DownloadClientBase<PneumaticSettings>
{
private readonly IHttpClient _httpClient;
public Pneumatic(IHttpClient httpClient,
IConfigService configService,
public Pneumatic(IConfigService configService,
IDiskProvider diskProvider,
Logger logger)
: base(configService, diskProvider, logger)
{
_httpClient = httpClient;
}
public override string Name => "Pneumatic";
public override DownloadProtocol Protocol => DownloadProtocol.Usenet;
public override string Download(ReleaseInfo release, bool redirect)
public override async Task<string> Download(ReleaseInfo release, bool redirect, IIndexer indexer)
{
var url = release.DownloadUrl;
var url = new Uri(release.DownloadUrl);
var title = release.Title;
title = StringUtil.CleanFileName(title);
@@ -40,7 +38,10 @@ namespace NzbDrone.Core.Download.Clients.Pneumatic
var nzbFile = Path.Combine(Settings.NzbFolder, title + ".nzb");
_logger.Debug("Downloading NZB from: {0} to: {1}", url, nzbFile);
_httpClient.DownloadFile(url, nzbFile);
var nzbData = await indexer.Download(url);
File.WriteAllBytes(nzbFile, nzbData);
_logger.Debug("NZB Download succeeded, saved to: {0}", nzbFile);

3
src/NzbDrone.Core/Download/Clients/Sabnzbd/Sabnzbd.cs
View File

@@ -22,9 +22,8 @@ namespace NzbDrone.Core.Download.Clients.Sabnzbd
IHttpClient httpClient,
IConfigService configService,
IDiskProvider diskProvider,
IValidateNzbs nzbValidationService,
Logger logger)
: base(httpClient, configService, diskProvider, nzbValidationService, logger)
: base(httpClient, configService, diskProvider, logger)
{
_proxy = proxy;
}

2
src/NzbDrone.Core/Download/Clients/rTorrent/RTorrent.cs
View File

@@ -66,7 +66,7 @@ namespace NzbDrone.Core.Download.Clients.RTorrent
{
_logger.Debug("rTorrent didn't add the torrent within {0} seconds: {1}.", tries * retryDelay / 1000, filename);
throw new ReleaseDownloadException(release, "Downloading torrent failed");
throw new ReleaseDownloadException("Downloading torrent failed");
}
return hash;

3
src/NzbDrone.Core/Download/DownloadClientBase.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using FluentValidation.Results;
using NLog;
using NzbDrone.Common.Disk;
@@ -54,7 +55,7 @@ namespace NzbDrone.Core.Download
get;
}
public abstract string Download(ReleaseInfo release, bool redirect);
public abstract Task<string> Download(ReleaseInfo release, bool redirect, IIndexer indexer);
public ValidationResult Test()
{

2
src/NzbDrone.Core/Indexers/DownloadMappingService.cs → src/NzbDrone.Core/Download/DownloadMappingService.cs
View File

@@ -6,7 +6,7 @@ using NzbDrone.Common.Extensions;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Security;
namespace NzbDrone.Core.Indexers
namespace NzbDrone.Core.Download
{
public interface IDownloadMappingService
{

8
src/NzbDrone.Core/Download/DownloadService.cs
View File

@@ -17,7 +17,7 @@ namespace NzbDrone.Core.Download
{
public interface IDownloadService
{
void SendReportToClient(ReleaseInfo release, string source, string host, bool redirect);
Task SendReportToClient(ReleaseInfo release, string source, string host, bool redirect);
Task<byte[]> DownloadReport(string link, int indexerId, string source, string host, string title);
void RecordRedirect(string link, int indexerId, string source, string host, string title);
}
@@ -49,7 +49,7 @@ namespace NzbDrone.Core.Download
_logger = logger;
}
public void SendReportToClient(ReleaseInfo release, string source, string host, bool redirect)
public async Task SendReportToClient(ReleaseInfo release, string source, string host, bool redirect)
{
var downloadTitle = release.Title;
var downloadClient = _downloadClientProvider.GetDownloadClient(release.DownloadProtocol);
@@ -69,10 +69,12 @@ namespace NzbDrone.Core.Download
_rateLimitService.WaitAndPulse(url.Host, TimeSpan.FromSeconds(2));
}
var indexer = _indexerFactory.GetInstance(_indexerFactory.Get(release.IndexerId));
string downloadClientId;
try
{
downloadClientId = downloadClient.Download(release, redirect);
downloadClientId = await downloadClient.Download(release, redirect, indexer);
_downloadClientStatusService.RecordSuccess(downloadClient.Definition.Id);
_indexerStatusService.RecordSuccess(release.IndexerId);
}

3
src/NzbDrone.Core/Download/IDownloadClient.cs
View File

@@ -1,3 +1,4 @@
using System.Threading.Tasks;
using NzbDrone.Core.Indexers;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.ThingiProvider;
@@ -7,6 +8,6 @@ namespace NzbDrone.Core.Download
public interface IDownloadClient : IProvider
{
DownloadProtocol Protocol { get; }
string Download(ReleaseInfo release, bool redirect);
Task<string> Download(ReleaseInfo release, bool redirect, IIndexer indexer);
}
}

10
src/NzbDrone.Core/Download/NzbValidationService.cs
View File

@@ -8,12 +8,12 @@ namespace NzbDrone.Core.Download
{
public interface IValidateNzbs
{
void Validate(string filename, byte[] fileContent);
void Validate(byte[] fileContent);
}
public class NzbValidationService : IValidateNzbs
{
public void Validate(string filename, byte[] fileContent)
public void Validate(byte[] fileContent)
{
var reader = new StreamReader(new MemoryStream(fileContent));
@@ -24,7 +24,7 @@ namespace NzbDrone.Core.Download
if (nzb == null)
{
throw new InvalidNzbException("Invalid NZB: No Root element [{0}]", filename);
throw new InvalidNzbException("Invalid NZB: No Root element");
}
// nZEDb has an bug in their error reporting code spitting out invalid http status codes
@@ -37,7 +37,7 @@ namespace NzbDrone.Core.Download
if (!nzb.Name.LocalName.Equals("nzb"))
{
throw new InvalidNzbException("Invalid NZB: Unexpected root element. Expected 'nzb' found '{0}' [{1}]", nzb.Name.LocalName, filename);
throw new InvalidNzbException("Invalid NZB: Unexpected root element. Expected 'nzb' found '{0}'", nzb.Name.LocalName);
}
var ns = nzb.Name.Namespace;
@@ -45,7 +45,7 @@ namespace NzbDrone.Core.Download
if (files.Empty())
{
throw new InvalidNzbException("Invalid NZB: No files [{0}]", filename);
throw new InvalidNzbException("Invalid NZB: No files");
}
}
}

72
src/NzbDrone.Core/Download/TorrentClientBase.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Net;
using System.Threading.Tasks;
using MonoTorrent;
using NLog;
using NzbDrone.Common.Disk;
@@ -39,7 +40,7 @@ namespace NzbDrone.Core.Download
protected abstract string AddFromTorrentFile(ReleaseInfo release, string hash, string filename, byte[] fileContent);
protected abstract string AddFromTorrentLink(ReleaseInfo release, string hash, string torrentLink);
public override string Download(ReleaseInfo release, bool redirect)
public override async Task<string> Download(ReleaseInfo release, bool redirect, IIndexer indexer)
{
var torrentInfo = release as TorrentInfo;
@@ -66,7 +67,7 @@ namespace NzbDrone.Core.Download
{
try
{
return DownloadFromWebUrl(release, torrentUrl);
return await DownloadFromWebUrl(release, indexer, torrentUrl);
}
catch (Exception ex)
{
@@ -87,7 +88,7 @@ namespace NzbDrone.Core.Download
}
catch (NotSupportedException ex)
{
throw new ReleaseDownloadException(release, "Magnet not supported by download client. ({0})", ex.Message);
throw new ReleaseDownloadException("Magnet not supported by download client. ({0})", ex.Message);
}
}
}
@@ -103,7 +104,7 @@ namespace NzbDrone.Core.Download
{
if (torrentUrl.IsNullOrWhiteSpace())
{
throw new ReleaseDownloadException(release, "Magnet not supported by download client. ({0})", ex.Message);
throw new ReleaseDownloadException("Magnet not supported by download client. ({0})", ex.Message);
}
_logger.Debug("Magnet not supported by download client, trying torrent. ({0})", ex.Message);
@@ -112,75 +113,18 @@ namespace NzbDrone.Core.Download
if (torrentUrl.IsNotNullOrWhiteSpace())
{
return DownloadFromWebUrl(release, torrentUrl);
return await DownloadFromWebUrl(release, indexer, torrentUrl);
}
}
return null;
}
private string DownloadFromWebUrl(ReleaseInfo release, string torrentUrl)
private async Task<string> DownloadFromWebUrl(ReleaseInfo release, IIndexer indexer, string torrentUrl)
{
byte[] torrentFile = null;
try
{
var request = new HttpRequest(torrentUrl);
request.Headers.Accept = "application/x-bittorrent";
request.AllowAutoRedirect = false;
var response = _httpClient.Get(request);
if (response.StatusCode == HttpStatusCode.MovedPermanently ||
response.StatusCode == HttpStatusCode.Found ||
response.StatusCode == HttpStatusCode.SeeOther)
{
var locationHeader = response.Headers.GetSingleValue("Location");
_logger.Trace("Torrent request is being redirected to: {0}", locationHeader);
if (locationHeader != null)
{
if (locationHeader.StartsWith("magnet:"))
{
return DownloadFromMagnetUrl(release, locationHeader);
}
return DownloadFromWebUrl(release, locationHeader);
}
throw new WebException("Remote website tried to redirect without providing a location.");
}
torrentFile = response.ResponseData;
_logger.Debug("Downloading torrent for release '{0}' finished ({1} bytes from {2})", release.Title, torrentFile.Length, torrentUrl);
}
catch (HttpException ex)
{
if (ex.Response.StatusCode == HttpStatusCode.NotFound)
{
_logger.Error(ex, "Downloading torrent file for release '{0}' failed since it no longer exists ({1})", release.Title, torrentUrl);
throw new ReleaseUnavailableException(release, "Downloading torrent failed", ex);
}
if ((int)ex.Response.StatusCode == 429)
{
_logger.Error("API Grab Limit reached for {0}", torrentUrl);
}
else
{
_logger.Error(ex, "Downloading torrent file for release '{0}' failed ({1})", release.Title, torrentUrl);
}
throw new ReleaseDownloadException(release, "Downloading torrent failed", ex);
}
catch (WebException ex)
{
_logger.Error(ex, "Downloading torrent file for release '{0}' failed ({1})", release.Title, torrentUrl);
throw new ReleaseDownloadException(release, "Downloading torrent failed", ex);
}
torrentFile = await indexer.Download(new Uri(torrentUrl));
var filename = string.Format("{0}.torrent", StringUtil.CleanFileName(release.Title));
var hash = _torrentFileInfoReader.GetHashFromTorrentFile(torrentFile);

46
src/NzbDrone.Core/Download/UsenetClientBase.cs
View File

@@ -1,9 +1,9 @@
using System.Net;
using System;
using System.Threading.Tasks;
using NLog;
using NzbDrone.Common.Disk;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Exceptions;
using NzbDrone.Core.Indexers;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
@@ -15,17 +15,14 @@ namespace NzbDrone.Core.Download
where TSettings : IProviderConfig, new()
{
protected readonly IHttpClient _httpClient;
private readonly IValidateNzbs _nzbValidationService;
protected UsenetClientBase(IHttpClient httpClient,
IConfigService configService,
IDiskProvider diskProvider,
IValidateNzbs nzbValidationService,
Logger logger)
: base(configService, diskProvider, logger)
{
_httpClient = httpClient;
_nzbValidationService = nzbValidationService;
}
public override DownloadProtocol Protocol => DownloadProtocol.Usenet;
@@ -33,9 +30,9 @@ namespace NzbDrone.Core.Download
protected abstract string AddFromNzbFile(ReleaseInfo release, string filename, byte[] fileContents);
protected abstract string AddFromLink(ReleaseInfo release);
public override string Download(ReleaseInfo release, bool redirect)
public override async Task<string> Download(ReleaseInfo release, bool redirect, IIndexer indexer)
{
var url = release.DownloadUrl;
var url = new Uri(release.DownloadUrl);
if (redirect)
{
@@ -46,40 +43,7 @@ namespace NzbDrone.Core.Download
byte[] nzbData;
try
{
var request = new HttpRequest(url);
nzbData = _httpClient.Get(request).ResponseData;
_logger.Debug("Downloaded nzb for release '{0}' finished ({1} bytes from {2})", release.Title, nzbData.Length, url);
}
catch (HttpException ex)
{
if (ex.Response.StatusCode == HttpStatusCode.NotFound)
{
_logger.Error(ex, "Downloading nzb file for release '{0}' failed since it no longer exists ({1})", release.Title, url);
throw new ReleaseUnavailableException(release, "Downloading nzb failed", ex);
}
if ((int)ex.Response.StatusCode == 429)
{
_logger.Error("API Grab Limit reached for {0}", url);
}
else
{
_logger.Error(ex, "Downloading nzb for release '{0}' failed ({1})", release.Title, url);
}
throw new ReleaseDownloadException(release, "Downloading nzb failed", ex);
}
catch (WebException ex)
{
_logger.Error(ex, "Downloading nzb for release '{0}' failed ({1})", release.Title, url);
throw new ReleaseDownloadException(release, "Downloading nzb failed", ex);
}
_nzbValidationService.Validate(filename, nzbData);
nzbData = await indexer.Download(url);
_logger.Info("Adding report [{0}] to the queue.", release.Title);
return AddFromNzbFile(release, filename, nzbData);

15
src/NzbDrone.Core/Exceptions/DownloadClientRejectedReleaseException.cs
View File

@@ -1,28 +1,33 @@
using System;
using System;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Exceptions
{
public class DownloadClientRejectedReleaseException : ReleaseDownloadException
{
public ReleaseInfo Release { get; set; }
public DownloadClientRejectedReleaseException(ReleaseInfo release, string message, params object[] args)
: base(release, message, args)
: base(message, args)
{
Release = release;
}
public DownloadClientRejectedReleaseException(ReleaseInfo release, string message)
: base(release, message)
: base(message)
{
Release = release;
}
public DownloadClientRejectedReleaseException(ReleaseInfo release, string message, Exception innerException, params object[] args)
: base(release, message, innerException, args)
: base(message, innerException, args)
{
Release = release;
}
public DownloadClientRejectedReleaseException(ReleaseInfo release, string message, Exception innerException)
: base(release, message, innerException)
: base(message, innerException)
{
Release = release;
}
}
}

16
src/NzbDrone.Core/Exceptions/ReleaseDownloadException.cs
View File

@@ -1,4 +1,4 @@
using System;
using System;
using NzbDrone.Common.Exceptions;
using NzbDrone.Core.Parser.Model;
@@ -6,30 +6,24 @@ namespace NzbDrone.Core.Exceptions
{
public class ReleaseDownloadException : NzbDroneException
{
public ReleaseInfo Release { get; set; }
public ReleaseDownloadException(ReleaseInfo release, string message, params object[] args)
public ReleaseDownloadException(string message, params object[] args)
: base(message, args)
{
Release = release;
}
public ReleaseDownloadException(ReleaseInfo release, string message)
public ReleaseDownloadException(string message)
: base(message)
{
Release = release;
}
public ReleaseDownloadException(ReleaseInfo release, string message, Exception innerException, params object[] args)
public ReleaseDownloadException(string message, Exception innerException, params object[] args)
: base(message, innerException, args)
{
Release = release;
}
public ReleaseDownloadException(ReleaseInfo release, string message, Exception innerException)
public ReleaseDownloadException(string message, Exception innerException)
: base(message, innerException)
{
Release = release;
}
}
}

18
src/NzbDrone.Core/Exceptions/ReleaseUnavailableException.cs
View File

@@ -1,27 +1,27 @@
using System;
using System;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Exceptions
{
public class ReleaseUnavailableException : ReleaseDownloadException
{
public ReleaseUnavailableException(ReleaseInfo release, string message, params object[] args)
: base(release, message, args)
public ReleaseUnavailableException(string message, params object[] args)
: base(message, args)
{
}
public ReleaseUnavailableException(ReleaseInfo release, string message)
: base(release, message)
public ReleaseUnavailableException(string message)
: base(message)
{
}
public ReleaseUnavailableException(ReleaseInfo release, string message, Exception innerException, params object[] args)
: base(release, message, innerException, args)
public ReleaseUnavailableException(string message, Exception innerException, params object[] args)
: base(message, innerException, args)
{
}
public ReleaseUnavailableException(ReleaseInfo release, string message, Exception innerException)
: base(release, message, innerException)
public ReleaseUnavailableException(string message, Exception innerException)
: base(message, innerException)
{
}
}

1
src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs
View File

@@ -4,6 +4,7 @@ using System.Linq;
using System.Threading.Tasks;
using NLog;
using NzbDrone.Common.Instrumentation.Extensions;
using NzbDrone.Core.Download;
using NzbDrone.Core.Indexers;
using NzbDrone.Core.Indexers.Events;
using NzbDrone.Core.IndexerSearch.Definitions;

2
src/NzbDrone.Core/Indexers/Definitions/AnimeBytes.cs
View File

@@ -23,7 +23,7 @@ using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Definitions
{
public class AnimeBytes : HttpIndexerBase<AnimeBytesSettings>
public class AnimeBytes : TorrentIndexerBase<AnimeBytesSettings>
{
public override string Name => "AnimeBytes";
public override string BaseUrl => "https://animebytes.tv/";

2
src/NzbDrone.Core/Indexers/Definitions/AnimeTorrents.cs
View File

@@ -21,7 +21,7 @@ using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Definitions
{
public class AnimeTorrents : HttpIndexerBase<AnimeTorrentsSettings>
public class AnimeTorrents : TorrentIndexerBase<AnimeTorrentsSettings>
{
public override string Name => "AnimeTorrents";

2
src/NzbDrone.Core/Indexers/Definitions/Avistaz/AvistazBase.cs
View File

@@ -9,7 +9,7 @@ using NzbDrone.Core.Messaging.Events;
namespace NzbDrone.Core.Indexers.Definitions.Avistaz
{
public abstract class AvistazBase : HttpIndexerBase<AvistazSettings>
public abstract class AvistazBase : TorrentIndexerBase<AvistazSettings>
{
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
public override string BaseUrl => "";

2
src/NzbDrone.Core/Indexers/Definitions/BakaBT.cs
View File

@@ -21,7 +21,7 @@ using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Definitions
{
public class BakaBT : HttpIndexerBase<BakaBTSettings>
public class BakaBT : TorrentIndexerBase<BakaBTSettings>
{
public override string Name => "BakaBT";

2
src/NzbDrone.Core/Indexers/Definitions/BeyondHD.cs
View File

@@ -21,7 +21,7 @@ using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Definitions
{
public class BeyondHD : HttpIndexerBase<BeyondHDSettings>
public class BeyondHD : TorrentIndexerBase<BeyondHDSettings>
{
public override string Name => "BeyondHD";

2
src/NzbDrone.Core/Indexers/Definitions/BroadcastheNet/BroadcastheNet.cs
View File

@@ -6,7 +6,7 @@ using NzbDrone.Core.Messaging.Events;
namespace NzbDrone.Core.Indexers.BroadcastheNet
{
public class BroadcastheNet : HttpIndexerBase<BroadcastheNetSettings>
public class BroadcastheNet : TorrentIndexerBase<BroadcastheNetSettings>
{
public override string Name => "BroadcasTheNet";

33
src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Net;
using System.Text;
using System.Threading.Tasks;
using FluentValidation.Results;
@@ -7,6 +8,7 @@ using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Exceptions;
using NzbDrone.Core.IndexerVersions;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.ThingiProvider;
@@ -14,7 +16,7 @@ using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Cardigann
{
public class Cardigann : HttpIndexerBase<CardigannSettings>
public class Cardigann : TorrentIndexerBase<CardigannSettings>
{
private readonly IIndexerDefinitionUpdateService _definitionService;
private readonly ICached<CardigannRequestGenerator> _generatorCache;
@@ -147,6 +149,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
if (request.Url.Scheme == "magnet")
{
ValidateMagnet(request.Url.FullUri);
return Encoding.UTF8.GetBytes(request.Url.FullUri);
}
@@ -159,10 +162,36 @@ namespace NzbDrone.Core.Indexers.Cardigann
var response = await _httpClient.ExecuteAsync(request);
downloadBytes = response.ResponseData;
}
catch (HttpException ex)
{
if (ex.Response.StatusCode == HttpStatusCode.NotFound)
{
_logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", request.Url.FullUri);
throw new ReleaseUnavailableException("Downloading torrent failed", ex);
}
if ((int)ex.Response.StatusCode == 429)
{
_logger.Error("API Grab Limit reached for {0}", request.Url.FullUri);
}
else
{
_logger.Error(ex, "Downloading torrent file for release failed ({0})", request.Url.FullUri);
}
throw new ReleaseDownloadException("Downloading torrent failed", ex);
}
catch (WebException ex)
{
_logger.Error(ex, "Downloading torrent file for release failed ({0})", request.Url.FullUri);
throw new ReleaseDownloadException("Downloading torrent failed", ex);
}
catch (Exception)
{
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Error("Download failed");
_logger.Error("Downloading torrent failed");
throw;
}
return downloadBytes;

2
src/NzbDrone.Core/Indexers/Definitions/DigitalCore.cs
View File

@@ -21,7 +21,7 @@ using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Definitions
{
public class DigitalCore : HttpIndexerBase<DigitalCoreSettings>
public class DigitalCore : TorrentIndexerBase<DigitalCoreSettings>
{
public override string Name => "DigitalCore";
public override string BaseUrl => "https://digitalcore.club/";

2
src/NzbDrone.Core/Indexers/Definitions/FileList/FileList.cs
View File

@@ -6,7 +6,7 @@ using NzbDrone.Core.Messaging.Events;
namespace NzbDrone.Core.Indexers.FileList
{
public class FileList : HttpIndexerBase<FileListSettings>
public class FileList : TorrentIndexerBase<FileListSettings>
{
public override string Name => "FileList.io";
public override string BaseUrl => "https://filelist.io";

2
src/NzbDrone.Core/Indexers/Definitions/Gazelle/Gazelle.cs
View File

@@ -7,7 +7,7 @@ using NzbDrone.Core.Messaging.Events;
namespace NzbDrone.Core.Indexers.Gazelle
{
public abstract class Gazelle : HttpIndexerBase<GazelleSettings>
public abstract class Gazelle : TorrentIndexerBase<GazelleSettings>
{
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
public override string BaseUrl => "";

2
src/NzbDrone.Core/Indexers/Definitions/HDBits/HDBits.cs
View File

@@ -6,7 +6,7 @@ using NzbDrone.Core.Messaging.Events;
namespace NzbDrone.Core.Indexers.HDBits
{
public class HDBits : HttpIndexerBase<HDBitsSettings>
public class HDBits : TorrentIndexerBase<HDBitsSettings>
{
public override string Name => "HDBits";
public override string BaseUrl => "https://hdbits.org";

2
src/NzbDrone.Core/Indexers/Definitions/HDTorrents.cs
View File

@@ -20,7 +20,7 @@ using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Definitions
{
public class HDTorrents : HttpIndexerBase<HDTorrentsSettings>
public class HDTorrents : TorrentIndexerBase<HDTorrentsSettings>
{
public override string Name => "HD-Torrents";

7
src/NzbDrone.Core/Indexers/Definitions/Headphones/Headphones.cs
View File

@@ -6,11 +6,12 @@ using FluentValidation.Results;
using NLog;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Download;
using NzbDrone.Core.Messaging.Events;
namespace NzbDrone.Core.Indexers.Headphones
{
public class Headphones : HttpIndexerBase<HeadphonesSettings>
public class Headphones : UsenetIndexerBase<HeadphonesSettings>
{
public override string Name => "Headphones VIP";
@@ -35,8 +36,8 @@ namespace NzbDrone.Core.Indexers.Headphones
return new HeadphonesRssParser(Capabilities.Categories);
}
public Headphones(IHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
: base(httpClient, eventAggregator, indexerStatusService, configService, logger)
public Headphones(IHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, IValidateNzbs nzbValidationService, Logger logger)
: base(httpClient, eventAggregator, indexerStatusService, configService, nzbValidationService, logger)
{
}

2
src/NzbDrone.Core/Indexers/Definitions/IPTorrents.cs
View File

@@ -18,7 +18,7 @@ using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Definitions
{
public class IPTorrents : HttpIndexerBase<IPTorrentsSettings>
public class IPTorrents : TorrentIndexerBase<IPTorrentsSettings>
{
public override string Name => "IPTorrents";

2
src/NzbDrone.Core/Indexers/Definitions/ImmortalSeed.cs
View File

@@ -21,7 +21,7 @@ using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Definitions
{
public class ImmortalSeed : HttpIndexerBase<ImmortalSeedSettings>
public class ImmortalSeed : TorrentIndexerBase<ImmortalSeedSettings>
{
public override string Name => "ImmortalSeed";

2
src/NzbDrone.Core/Indexers/Definitions/Milkie.cs
View File

@@ -16,7 +16,7 @@ using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Definitions
{
public class Milkie : HttpIndexerBase<MilkieSettings>
public class Milkie : TorrentIndexerBase<MilkieSettings>
{
public override string Name => "Milkie";

2
src/NzbDrone.Core/Indexers/Definitions/MyAnonamouse.cs
View File

@@ -18,7 +18,7 @@ using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Definitions
{
public class MyAnonamouse : HttpIndexerBase<MyAnonamouseSettings>
public class MyAnonamouse : TorrentIndexerBase<MyAnonamouseSettings>
{
public override string Name => "MyAnonamouse";

7
src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs
View File

@@ -7,13 +7,14 @@ using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Download;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.ThingiProvider;
using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Newznab
{
public class Newznab : HttpIndexerBase<NewznabSettings>
public class Newznab : UsenetIndexerBase<NewznabSettings>
{
private readonly INewznabCapabilitiesProvider _capabilitiesProvider;
@@ -108,8 +109,8 @@ namespace NzbDrone.Core.Indexers.Newznab
}
}
public Newznab(INewznabCapabilitiesProvider capabilitiesProvider, IHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
: base(httpClient, eventAggregator, indexerStatusService, configService, logger)
public Newznab(INewznabCapabilitiesProvider capabilitiesProvider, IHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, IValidateNzbs nzbValidationService, Logger logger)
: base(httpClient, eventAggregator, indexerStatusService, configService, nzbValidationService, logger)
{
_capabilitiesProvider = capabilitiesProvider;
}

2
src/NzbDrone.Core/Indexers/Definitions/PassThePopcorn/PassThePopcorn.cs
View File

@@ -7,7 +7,7 @@ using NzbDrone.Core.Messaging.Events;
namespace NzbDrone.Core.Indexers.PassThePopcorn
{
public class PassThePopcorn : HttpIndexerBase<PassThePopcornSettings>
public class PassThePopcorn : TorrentIndexerBase<PassThePopcornSettings>
{
public override string Name => "PassThePopcorn";
public override string BaseUrl => "https://passthepopcorn.me";

2
src/NzbDrone.Core/Indexers/Definitions/PreToMe.cs
View File

@@ -21,7 +21,7 @@ using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Definitions
{
public class PreToMe : HttpIndexerBase<PreToMeSettings>
public class PreToMe : TorrentIndexerBase<PreToMeSettings>
{
public override string Name => "PreToMe";
public override string BaseUrl => "https://pretome.info/";

2
src/NzbDrone.Core/Indexers/Definitions/Rarbg/Rarbg.cs
View File

@@ -11,7 +11,7 @@ using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Rarbg
{
public class Rarbg : HttpIndexerBase<RarbgSettings>
public class Rarbg : TorrentIndexerBase<RarbgSettings>
{
private readonly IRarbgTokenProvider _tokenProvider;

2
src/NzbDrone.Core/Indexers/Definitions/RevolutionTT.cs
View File

@@ -21,7 +21,7 @@ using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Definitions
{
public class RevolutionTT : HttpIndexerBase<RevolutionTTSettings>
public class RevolutionTT : TorrentIndexerBase<RevolutionTTSettings>
{
public override string Name => "RevolutionTT";

2
src/NzbDrone.Core/Indexers/Definitions/SubsPlease.cs
View File

@@ -18,7 +18,7 @@ using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Definitions
{
public class SubsPlease : HttpIndexerBase<SubsPleaseSettings>
public class SubsPlease : TorrentIndexerBase<SubsPleaseSettings>
{
public override string Name => "SubsPlease";
public override string BaseUrl => "https://subsplease.org/";

2
src/NzbDrone.Core/Indexers/Definitions/SuperBits.cs
View File

@@ -19,7 +19,7 @@ using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Definitions
{
public class SuperBits : HttpIndexerBase<SuperBitsSettings>
public class SuperBits : TorrentIndexerBase<SuperBitsSettings>
{
public override string Name => "SuperBits";

2
src/NzbDrone.Core/Indexers/Definitions/ThePirateBay.cs
View File

@@ -18,7 +18,7 @@ using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Definitions
{
public class ThePirateBay : HttpIndexerBase<ThePirateBaySettings>
public class ThePirateBay : TorrentIndexerBase<ThePirateBaySettings>
{
public override string Name => "ThePirateBay";
public override string BaseUrl => "https://thepiratebay.org/";

2
src/NzbDrone.Core/Indexers/Definitions/TorrentDay.cs
View File

@@ -17,7 +17,7 @@ using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Definitions
{
public class TorrentDay : HttpIndexerBase<TorrentDaySettings>
public class TorrentDay : TorrentIndexerBase<TorrentDaySettings>
{
public override string Name => "TorrentDay";

2
src/NzbDrone.Core/Indexers/Definitions/TorrentLeech.cs
View File

@@ -21,7 +21,7 @@ using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Definitions
{
public class TorrentLeech : HttpIndexerBase<TorrentLeechSettings>
public class TorrentLeech : TorrentIndexerBase<TorrentLeechSettings>
{
public override string Name => "TorrentLeech";

2
src/NzbDrone.Core/Indexers/Definitions/TorrentPotato/TorrentPotato.cs
View File

@@ -6,7 +6,7 @@ using NzbDrone.Core.Messaging.Events;
namespace NzbDrone.Core.Indexers.TorrentPotato
{
public class TorrentPotato : HttpIndexerBase<TorrentPotatoSettings>
public class TorrentPotato : TorrentIndexerBase<TorrentPotatoSettings>
{
public override string Name => "TorrentPotato";
public override string BaseUrl => "http://127.0.0.1";

2
src/NzbDrone.Core/Indexers/Definitions/TorrentSeeds.cs
View File

@@ -20,7 +20,7 @@ using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Definitions
{
public class TorrentSeeds : HttpIndexerBase<TorrentSeedsSettings>
public class TorrentSeeds : TorrentIndexerBase<TorrentSeedsSettings>
{
public override string Name => "TorrentSeeds";

2
src/NzbDrone.Core/Indexers/Definitions/Torznab/Torznab.cs
View File

@@ -14,7 +14,7 @@ using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Torznab
{
public class Torznab : HttpIndexerBase<TorznabSettings>
public class Torznab : TorrentIndexerBase<TorznabSettings>
{
private readonly INewznabCapabilitiesProvider _capabilitiesProvider;

2
src/NzbDrone.Core/Indexers/Definitions/UNIT3D/Unit3dBase.cs
View File

@@ -5,7 +5,7 @@ using NzbDrone.Core.Messaging.Events;
namespace NzbDrone.Core.Indexers.Definitions.UNIT3D
{
public abstract class Unit3dBase : HttpIndexerBase<Unit3dSettings>
public abstract class Unit3dBase : TorrentIndexerBase<Unit3dSettings>
{
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
public override string BaseUrl => "";

38
src/NzbDrone.Core/Indexers/HttpIndexerBase.cs
View File

@@ -1,6 +1,5 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Net;
using System.Text;
@@ -10,7 +9,6 @@ using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Exceptions;
using NzbDrone.Core.Http.CloudFlare;
using NzbDrone.Core.Indexers.Events;
using NzbDrone.Core.Indexers.Exceptions;
@@ -105,42 +103,6 @@ namespace NzbDrone.Core.Indexers
return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria));
}
public override async Task<byte[]> Download(Uri link)
{
Cookies = GetCookies();
if (link.Scheme == "magnet")
{
return Encoding.UTF8.GetBytes(link.OriginalString);
}
var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
if (Cookies != null)
{
requestBuilder.SetCookies(Cookies);
}
var request = requestBuilder.Build();
request.AllowAutoRedirect = FollowRedirect;
var downloadBytes = Array.Empty<byte>();
try
{
var response = await _httpClient.ExecuteAsync(request);
downloadBytes = response.ResponseData;
}
catch (Exception)
{
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Error("Download failed");
throw;
}
return downloadBytes;
}
protected IIndexerRequestGenerator SetCookieFunctions(IIndexerRequestGenerator generator)
{
//A func ensures cookies are always updated to the latest. This way, the first page could update the cookies and then can be reused by the second page.

2
src/NzbDrone.Core/Indexers/IndexerBase.cs
View File

@@ -77,7 +77,7 @@ namespace NzbDrone.Core.Indexers
public abstract Task<IndexerPageableQueryResult> Fetch(TvSearchCriteria searchCriteria);
public abstract Task<IndexerPageableQueryResult> Fetch(BookSearchCriteria searchCriteria);
public abstract Task<IndexerPageableQueryResult> Fetch(BasicSearchCriteria searchCriteria);
public abstract Task<byte[]> Download(Uri searchCriteria);
public abstract Task<byte[]> Download(Uri link);
public abstract IndexerCapabilities GetCapabilities();

90
src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs Normal file
View File

@@ -0,0 +1,90 @@
using System;
using System.Net;
using System.Text;
using System.Threading.Tasks;
using MonoTorrent;
using NLog;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Exceptions;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.ThingiProvider;
namespace NzbDrone.Core.Indexers
{
public abstract class TorrentIndexerBase<TSettings> : HttpIndexerBase<TSettings>
where TSettings : IProviderConfig, new()
{
protected TorrentIndexerBase(IHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
: base(httpClient, eventAggregator, indexerStatusService, configService, logger)
{
}
public override async Task<byte[]> Download(Uri link)
{
Cookies = GetCookies();
if (link.Scheme == "magnet")
{
ValidateMagnet(link.OriginalString);
return Encoding.UTF8.GetBytes(link.OriginalString);
}
var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
if (Cookies != null)
{
requestBuilder.SetCookies(Cookies);
}
var request = requestBuilder.Build();
request.AllowAutoRedirect = FollowRedirect;
byte[] torrentData;
try
{
var response = await _httpClient.ExecuteAsync(request);
torrentData = response.ResponseData;
}
catch (HttpException ex)
{
if (ex.Response.StatusCode == HttpStatusCode.NotFound)
{
_logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", link.AbsoluteUri);
throw new ReleaseUnavailableException("Downloading torrent failed", ex);
}
if ((int)ex.Response.StatusCode == 429)
{
_logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
}
else
{
_logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
}
throw new ReleaseDownloadException("Downloading torrent failed", ex);
}
catch (WebException ex)
{
_logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
throw new ReleaseDownloadException("Downloading torrent failed", ex);
}
catch (Exception)
{
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Error("Downloading torrent failed");
throw;
}
return torrentData;
}
protected void ValidateMagnet(string link)
{
MagnetLink.Parse(link);
}
}
}

85
src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs Normal file
View File

@@ -0,0 +1,85 @@
using System;
using System.Net;
using System.Threading.Tasks;
using NLog;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Download;
using NzbDrone.Core.Exceptions;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.ThingiProvider;
namespace NzbDrone.Core.Indexers
{
public abstract class UsenetIndexerBase<TSettings> : HttpIndexerBase<TSettings>
where TSettings : IProviderConfig, new()
{
private readonly IValidateNzbs _nzbValidationService;
protected UsenetIndexerBase(IHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, IValidateNzbs nzbValidationService, Logger logger)
: base(httpClient, eventAggregator, indexerStatusService, configService, logger)
{
_nzbValidationService = nzbValidationService;
}
public override async Task<byte[]> Download(Uri link)
{
Cookies = GetCookies();
var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
if (Cookies != null)
{
requestBuilder.SetCookies(Cookies);
}
var request = requestBuilder.Build();
request.AllowAutoRedirect = FollowRedirect;
byte[] nzbData;
try
{
var response = await _httpClient.ExecuteAsync(request);
nzbData = response.ResponseData;
_logger.Debug("Downloaded nzb for release finished ({0} bytes from {1})", nzbData.Length, link.AbsoluteUri);
}
catch (HttpException ex)
{
if (ex.Response.StatusCode == HttpStatusCode.NotFound)
{
_logger.Error(ex, "Downloading nzb file for release failed since it no longer exists ({0})", link.AbsoluteUri);
throw new ReleaseUnavailableException("Downloading nzb failed", ex);
}
if ((int)ex.Response.StatusCode == 429)
{
_logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
}
else
{
_logger.Error(ex, "Downloading nzb for release failed ({0})", link.AbsoluteUri);
}
throw new ReleaseDownloadException("Downloading nzb failed", ex);
}
catch (WebException ex)
{
_logger.Error(ex, "Downloading nzb for release failed ({0})", link.AbsoluteUri);
throw new ReleaseDownloadException("Downloading nzb failed", ex);
}
catch (Exception)
{
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Error("Downloading nzb failed");
throw;
}
_nzbValidationService.Validate(nzbData);
return nzbData;
}
}
}