shpaq/Prowlarr-Prowlarr
1
0
Fork 0
mirror of https://github.com/Prowlarr/Prowlarr.git synced 2025-09-17 17:14:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

Misc definition handling improvements

Browse Source
This commit is contained in:
Qstick
2022-01-30 18:49:58 -06:00
parent a3de574de5
commit 17608cf915
13 changed files with 302 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
frontend/src/Indexer/Index/Table/IndexerIndexRow.js
View File

@@ -244,12 +244,15 @@ class IndexerIndexRow extends Component {
onPress={this.onIndexerInfoPress}
/>
{
indexerUrls ?
<IconButton
className={styles.externalLink}
name={icons.EXTERNAL_LINK}
title={translate('Website')}
to={indexerUrls[0].replace('api.', '')}
/>
/> : null
}
<IconButton
name={icons.EDIT}
@@ -289,7 +292,7 @@ class IndexerIndexRow extends Component {
IndexerIndexRow.propTypes = {
id: PropTypes.number.isRequired,
indexerUrls: PropTypes.arrayOf(PropTypes.string).isRequired,
indexerUrls: PropTypes.arrayOf(PropTypes.string),
protocol: PropTypes.string.isRequired,
privacy: PropTypes.string.isRequired,
priority: PropTypes.number.isRequired,
@@ -298,7 +301,7 @@ IndexerIndexRow.propTypes = {
redirect: PropTypes.bool.isRequired,
appProfile: PropTypes.object.isRequired,
status: PropTypes.object,
capabilities: PropTypes.object.isRequired,
capabilities: PropTypes.object,
added: PropTypes.string.isRequired,
tags: PropTypes.arrayOf(PropTypes.number).isRequired,
columns: PropTypes.arrayOf(PropTypes.object).isRequired,

18
src/NzbDrone.Core/Datastore/Migration/015_indexer_versions.cs Normal file
View File

@@ -0,0 +1,18 @@
using FluentMigrator;
using NzbDrone.Core.Datastore.Migration.Framework;
namespace NzbDrone.Core.Datastore.Migration
{
[Migration(15)]
public class IndexerVersions : NzbDroneMigrationBase
{
protected override void MainDbUpgrade()
{
Create.TableForModel("IndexerDefinitionVersions")
.WithColumn("DefinitionId").AsString().NotNullable().Unique()
.WithColumn("File").AsString().NotNullable().Unique()
.WithColumn("Sha").AsString().Nullable()
.WithColumn("LastUpdated").AsDateTime().Nullable();
}
}
}

2
src/NzbDrone.Core/Datastore/TableMapping.cs
View File

@@ -11,6 +11,7 @@ using NzbDrone.Core.Datastore.Converters;
using NzbDrone.Core.Download;
using NzbDrone.Core.IndexerProxies;
using NzbDrone.Core.Indexers;
using NzbDrone.Core.IndexerVersions;
using NzbDrone.Core.Instrumentation;
using NzbDrone.Core.Jobs;
using NzbDrone.Core.Languages;
@@ -94,6 +95,7 @@ namespace NzbDrone.Core.Datastore
Mapper.Entity<UpdateHistory>("UpdateHistory").RegisterModel();
Mapper.Entity<AppSyncProfile>("AppSyncProfiles").RegisterModel();
Mapper.Entity<IndexerDefinitionVersion>("IndexerDefinitionVersions").RegisterModel();
}
private static void RegisterMappers()

48
src/NzbDrone.Core/HealthCheck/Checks/NoDefinitionCheck.cs Normal file
View File

@@ -0,0 +1,48 @@
using System.Linq;
using NLog;
using NzbDrone.Core.Indexers;
using NzbDrone.Core.Indexers.Cardigann;
using NzbDrone.Core.IndexerVersions;
using NzbDrone.Core.Localization;
using NzbDrone.Core.ThingiProvider.Events;
namespace NzbDrone.Core.HealthCheck.Checks
{
[CheckOn(typeof(ProviderDeletedEvent<IIndexer>))]
public class NoDefinitionCheck : HealthCheckBase
{
private readonly IIndexerDefinitionUpdateService _indexerDefinitionUpdateService;
private readonly IIndexerFactory _indexerFactory;
public NoDefinitionCheck(IIndexerDefinitionUpdateService indexerDefinitionUpdateService, IIndexerFactory indexerFactory, ILocalizationService localizationService)
: base(localizationService)
{
_indexerDefinitionUpdateService = indexerDefinitionUpdateService;
_indexerFactory = indexerFactory;
}
public override HealthCheck Check()
{
var currentDefs = _indexerDefinitionUpdateService.All();
var noDefIndexers = _indexerFactory.AllProviders(false)
.Where(i => i.Definition.Implementation == "Cardigann" && !currentDefs.Any(d => d.File == ((CardigannSettings)i.Definition.Settings).DefinitionFile)).ToList();
if (noDefIndexers.Count == 0)
{
return new HealthCheck(GetType());
}
var healthType = HealthCheckResult.Error;
var healthMessage = string.Format(_localizationService.GetLocalizedString("IndexerNoDefCheckMessage"),
string.Join(", ", noDefIndexers.Select(v => v.Definition.Name)));
return new HealthCheck(GetType(),
healthType,
healthMessage,
"#indexers-have-no-definition");
}
public override bool CheckOnSchedule => false;
}
}

9
src/NzbDrone.Core/HealthCheck/Checks/OutdatedDefinitionCheck.cs
View File

@@ -35,10 +35,13 @@ namespace NzbDrone.Core.HealthCheck.Checks
return new HealthCheck(GetType());
}
var healthType = HealthCheckResult.Warning;
var healthMessage = string.Format(_localizationService.GetLocalizedString("IndexerObsoleteCheckMessage"),
string.Join(", ", oldIndexers.Select(v => v.Definition.Name)));
return new HealthCheck(GetType(),
HealthCheckResult.Warning,
string.Format(_localizationService.GetLocalizedString("IndexerObsoleteCheckMessage"),
string.Join(", ", oldIndexers.Select(v => v.Definition.Name))),
healthType,
healthMessage,
"#indexers-are-obsolete");
}

147
src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs
View File

@@ -1,13 +1,11 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.Disk;
using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Indexers.Cardigann;
using NzbDrone.Core.Messaging.Commands;
@@ -19,7 +17,7 @@ namespace NzbDrone.Core.IndexerVersions
public interface IIndexerDefinitionUpdateService
{
List<CardigannMetaDefinition> All();
CardigannDefinition GetDefinition(string fileKey);
CardigannDefinition GetCachedDefinition(string fileKey);
List<string> GetBlocklist();
}
@@ -29,6 +27,8 @@ namespace NzbDrone.Core.IndexerVersions
private const string DEFINITION_BRANCH = "master";
private const int DEFINITION_VERSION = 3;
//Used when moving yml to C#
private readonly List<string> _defintionBlocklist = new List<string>()
{
"aither",
@@ -51,6 +51,7 @@ namespace NzbDrone.Core.IndexerVersions
private readonly IHttpClient _httpClient;
private readonly IAppFolderInfo _appFolderInfo;
private readonly IDiskProvider _diskProvider;
private readonly IIndexerDefinitionVersionService _versionService;
private readonly ICached<CardigannDefinition> _cache;
private readonly Logger _logger;
@@ -62,11 +63,13 @@ namespace NzbDrone.Core.IndexerVersions
public IndexerDefinitionUpdateService(IHttpClient httpClient,
IAppFolderInfo appFolderInfo,
IDiskProvider diskProvider,
IIndexerDefinitionVersionService versionService,
ICacheManager cacheManager,
Logger logger)
{
_appFolderInfo = appFolderInfo;
_diskProvider = diskProvider;
_versionService = versionService;
_cache = cacheManager.GetCache<CardigannDefinition>(typeof(CardigannDefinition), "definitions");
_httpClient = httpClient;
_logger = logger;
@@ -76,23 +79,65 @@ namespace NzbDrone.Core.IndexerVersions
{
var indexerList = new List<CardigannMetaDefinition>();
try
{
// Grab latest def list from server or fallback to disk
try
{
var request = new HttpRequest($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}");
var response = _httpClient.Get<List<CardigannMetaDefinition>>(request);
indexerList = response.Resource.Where(i => !_defintionBlocklist.Contains(i.File)).ToList();
}
catch
{
var definitionFolder = Path.Combine(_appFolderInfo.AppDataFolder, "Definitions");
var definitionFolder = Path.Combine(_appFolderInfo.AppDataFolder, "Definitions", "Custom");
indexerList = ReadDefinitionsFromDisk(indexerList, definitionFolder);
}
var directoryInfo = new DirectoryInfo(definitionFolder);
//Check for custom definitions
var customDefinitionFolder = Path.Combine(_appFolderInfo.AppDataFolder, "Definitions", "Custom");
indexerList = ReadDefinitionsFromDisk(indexerList, customDefinitionFolder);
}
catch
{
_logger.Error("Failed to Connect to Indexer Definition Server for Indexer listing");
}
return indexerList;
}
public CardigannDefinition GetCachedDefinition(string fileKey)
{
if (string.IsNullOrEmpty(fileKey))
{
throw new ArgumentNullException(nameof(fileKey));
}
var definition = _cache.Get(fileKey, () => GetUncachedDefinition(fileKey));
return definition;
}
public List<string> GetBlocklist()
{
return _defintionBlocklist;
}
private List<CardigannMetaDefinition> ReadDefinitionsFromDisk(List<CardigannMetaDefinition> defs, string path, SearchOption options = SearchOption.TopDirectoryOnly)
{
var indexerList = defs;
var directoryInfo = new DirectoryInfo(path);
if (directoryInfo.Exists)
{
var files = directoryInfo.GetFiles($"*.yml");
var files = directoryInfo.GetFiles($"*.yml", options);
foreach (var file in files)
{
_logger.Debug("Loading Custom Cardigann definition " + file.FullName);
_logger.Debug("Loading definition " + file.FullName);
try
{
@@ -103,7 +148,7 @@ namespace NzbDrone.Core.IndexerVersions
if (indexerList.Any(i => i.File == definition.File || i.Name == definition.Name))
{
_logger.Warn("Custom Cardigann definition {0} does not have unique file name or Indexer name", file.FullName);
_logger.Warn("Definition {0} does not have unique file name or Indexer name", file.FullName);
continue;
}
@@ -111,45 +156,15 @@ namespace NzbDrone.Core.IndexerVersions
}
catch (Exception e)
{
_logger.Error($"Error while parsing custom Cardigann definition {file.FullName}\n{e}");
_logger.Error($"Error while parsing Cardigann definition {file.FullName}\n{e}");
}
}
}
}
catch
{
_logger.Error("Failed to Connect to Indexer Definition Server for Indexer listing");
}
return indexerList;
}
public CardigannDefinition GetDefinition(string file)
{
if (string.IsNullOrEmpty(file))
{
throw new ArgumentNullException(nameof(file));
}
var definition = _cache.Get(file, () => LoadIndexerDef(file));
return definition;
}
public List<string> GetBlocklist()
{
return _defintionBlocklist;
}
private CardigannDefinition GetHttpDefinition(string id)
{
var req = new HttpRequest($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}/{id}");
var response = _httpClient.Get(req);
var definition = _deserializer.Deserialize<CardigannDefinition>(response.Content);
return CleanIndexerDefinition(definition);
}
private CardigannDefinition LoadIndexerDef(string fileKey)
private CardigannDefinition GetUncachedDefinition(string fileKey)
{
if (string.IsNullOrEmpty(fileKey))
{
@@ -184,9 +199,24 @@ namespace NzbDrone.Core.IndexerVersions
}
}
//Check to ensure it's in versioned defs before we go to web
if (!_versionService.All().Any(x => x.File == fileKey))
{
throw new ArgumentNullException(nameof(fileKey));
}
//No definition was returned locally, go to the web
return GetHttpDefinition(fileKey);
}
private CardigannDefinition GetHttpDefinition(string id)
{
var req = new HttpRequest($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}/{id}");
var response = _httpClient.Get(req);
var definition = _deserializer.Deserialize<CardigannDefinition>(response.Content);
return CleanIndexerDefinition(definition);
}
private CardigannDefinition CleanIndexerDefinition(CardigannDefinition definition)
{
if (definition.Settings == null)
@@ -242,29 +272,44 @@ namespace NzbDrone.Core.IndexerVersions
{
var startupFolder = _appFolderInfo.AppDataFolder;
var request = new HttpRequest($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}");
var response = _httpClient.Get<List<CardigannMetaDefinition>>(request);
var currentDefs = _versionService.All().ToDictionary(x => x.DefinitionId, x => x.Sha);
try
{
EnsureDefinitionsFolder();
var definitionsFolder = Path.Combine(startupFolder, "Definitions");
var saveFile = Path.Combine(startupFolder, "Definitions", $"indexers.zip");
_httpClient.DownloadFile($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}/package.zip", saveFile);
using (ZipArchive archive = ZipFile.OpenRead(saveFile))
foreach (var def in response.Resource)
{
archive.ExtractToDirectory(definitionsFolder, true);
try
{
var saveFile = Path.Combine(startupFolder, "Definitions", $"{def.File}.yml");
if (currentDefs.TryGetValue(def.Id, out var defSha) && defSha == def.Sha)
{
_logger.Trace("Indexer already up to date: {0}", def.File);
continue;
}
_diskProvider.DeleteFile(saveFile);
_httpClient.DownloadFile($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}/{def.File}", saveFile);
_cache.Clear();
_versionService.Upsert(new IndexerDefinitionVersion { Sha = def.Sha, DefinitionId = def.Id, File = def.File, LastUpdated = DateTime.UtcNow });
_logger.Debug("Updated indexer definitions");
_cache.Remove(def.File);
_logger.Debug("Updated definition: {0}", def.File);
}
catch (Exception ex)
{
_logger.Error(ex, "Definition update failed");
_logger.Error("Definition download failed: {0}, {1}", def.File, ex.Message);
}
}
}
catch (Exception ex)
{
_logger.Error(ex, "Definition download failed, error creating definitions folder in {0}", startupFolder);
}
}
}

13
src/NzbDrone.Core/IndexerVersions/IndexerDefinitionVersion.cs Normal file
View File

@@ -0,0 +1,13 @@
using System;
using NzbDrone.Core.Datastore;
namespace NzbDrone.Core.IndexerVersions
{
public class IndexerDefinitionVersion : ModelBase
{
public string File { get; set; }
public string Sha { get; set; }
public DateTime LastUpdated { get; set; }
public string DefinitionId { get; set; }
}
}

25
src/NzbDrone.Core/IndexerVersions/IndexerDefinitionVersionRepository.cs Normal file
View File

@@ -0,0 +1,25 @@
using System;
using System.Linq;
using NzbDrone.Core.Datastore;
using NzbDrone.Core.Messaging.Events;
namespace NzbDrone.Core.IndexerVersions
{
public interface IIndexerDefinitionVersionRepository : IBasicRepository<IndexerDefinitionVersion>
{
public IndexerDefinitionVersion GetByDefId(string defId);
}
public class IndexerDefinitionVersionRepository : BasicRepository<IndexerDefinitionVersion>, IIndexerDefinitionVersionRepository
{
public IndexerDefinitionVersionRepository(IMainDatabase database, IEventAggregator eventAggregator)
: base(database, eventAggregator)
{
}
public IndexerDefinitionVersion GetByDefId(string defId)
{
return Query(x => x.DefinitionId == defId).SingleOrDefault();
}
}
}

66
src/NzbDrone.Core/IndexerVersions/IndexerDefinitionVersionService.cs Normal file
View File

@@ -0,0 +1,66 @@
using System.Collections.Generic;
using System.Linq;
namespace NzbDrone.Core.IndexerVersions
{
public interface IIndexerDefinitionVersionService
{
IndexerDefinitionVersion Get(int indexerVersionId);
IndexerDefinitionVersion GetByDefId(string defId);
List<IndexerDefinitionVersion> All();
IndexerDefinitionVersion Add(IndexerDefinitionVersion defVersion);
IndexerDefinitionVersion Upsert(IndexerDefinitionVersion defVersion);
void Delete(int indexerVersionId);
}
public class IndexerDefinitionVersionService : IIndexerDefinitionVersionService
{
private readonly IIndexerDefinitionVersionRepository _repo;
public IndexerDefinitionVersionService(IIndexerDefinitionVersionRepository repo)
{
_repo = repo;
}
public IndexerDefinitionVersion Get(int indexerVersionId)
{
return _repo.Get(indexerVersionId);
}
public IndexerDefinitionVersion GetByDefId(string defId)
{
return _repo.GetByDefId(defId);
}
public List<IndexerDefinitionVersion> All()
{
return _repo.All().ToList();
}
public IndexerDefinitionVersion Add(IndexerDefinitionVersion defVersion)
{
_repo.Insert(defVersion);
return defVersion;
}
public IndexerDefinitionVersion Upsert(IndexerDefinitionVersion defVersion)
{
var existing = _repo.GetByDefId(defVersion.DefinitionId);
if (existing != null)
{
defVersion.Id = existing.Id;
}
defVersion = _repo.Upsert(defVersion);
return defVersion;
}
public void Delete(int indexerVersionId)
{
_repo.Delete(indexerVersionId);
}
}
}

4
src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs
View File

@@ -37,7 +37,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
{
var generator = _generatorCache.Get(Settings.DefinitionFile, () =>
new CardigannRequestGenerator(_configService,
_definitionService.GetDefinition(Settings.DefinitionFile),
_definitionService.GetCachedDefinition(Settings.DefinitionFile),
_logger)
{
HttpClient = _httpClient,
@@ -57,7 +57,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
public override IParseIndexerResponse GetParser()
{
return new CardigannParser(_configService,
_definitionService.GetDefinition(Settings.DefinitionFile),
_definitionService.GetCachedDefinition(Settings.DefinitionFile),
_logger)
{
Settings = Settings

4
src/NzbDrone.Core/Indexers/IndexerFactory.cs
View File

@@ -59,7 +59,7 @@ namespace NzbDrone.Core.Indexers
catch
{
// Skip indexer if we fail in Cardigann mapping
continue;
_logger.Debug("Indexer {0} has no definition", definition.Name);
}
}
@@ -96,7 +96,7 @@ namespace NzbDrone.Core.Indexers
private void MapCardigannDefinition(IndexerDefinition definition)
{
var settings = (CardigannSettings)definition.Settings;
var defFile = _definitionService.GetDefinition(settings.DefinitionFile);
var defFile = _definitionService.GetCachedDefinition(settings.DefinitionFile);
definition.ExtraFields = defFile.Settings;
if (defFile.Login?.Captcha != null && !definition.ExtraFields.Any(x => x.Type == "cardigannCaptcha"))

1
src/NzbDrone.Core/Localization/Core/en.json
View File

@@ -178,6 +178,7 @@
"IndexerLongTermStatusCheckAllClientMessage": "All indexers are unavailable due to failures for more than 6 hours",
"IndexerLongTermStatusCheckSingleClientMessage": "Indexers unavailable due to failures for more than 6 hours: {0}",
"IndexerObsoleteCheckMessage": "Indexers are obsolete or have been updated: {0}. Please remove and (or) re-add to Prowlarr",
"IndexerNoDefCheckMessage": "Indexers have no definition and will not work: {0}. Please remove and (or) re-add to Prowlarr",
"IndexerPriority": "Indexer Priority",
"IndexerPriorityHelpText": "Indexer Priority from 1 (Highest) to 50 (Lowest). Default: 25.",
"IndexerProxies": "Indexer Proxies",

2
src/Prowlarr.Api.V1/Indexers/IndexerResource.cs
View File

@@ -114,7 +114,7 @@ namespace Prowlarr.Api.V1.Indexers
var settings = (CardigannSettings)definition.Settings;
var cardigannDefinition = _definitionService.GetDefinition(settings.DefinitionFile);
var cardigannDefinition = _definitionService.GetCachedDefinition(settings.DefinitionFile);
foreach (var field in resource.Fields)
{