From a61d4ab88cf4635025a0f0bdea11d7b0f29363f0 Mon Sep 17 00:00:00 2001 From: Qstick Date: Thu, 24 Jun 2021 00:12:37 -0400 Subject: [PATCH] New: Stats filters --- frontend/src/Indexer/Stats/Stats.js | 27 ++- frontend/src/Indexer/Stats/StatsConnector.js | 21 ++- frontend/src/Indexer/Stats/StatsFilterMenu.js | 37 ++++ .../Stats/StatsFilterModalConnector.js | 24 +++ .../src/Store/Actions/indexerStatsActions.js | 121 ++++++++++++- .../History/HistoryRepository.cs | 8 + src/NzbDrone.Core/History/HistoryService.cs | 6 + .../IndexerStats/IndexerStatistics.cs | 8 + .../IndexerStatisticsRepository.cs | 103 ----------- .../IndexerStats/IndexerStatisticsService.cs | 169 ++++++++++++++++-- .../Indexers/IndexerStatsController.cs | 12 +- 11 files changed, 397 insertions(+), 139 deletions(-) create mode 100644 frontend/src/Indexer/Stats/StatsFilterMenu.js create mode 100644 frontend/src/Indexer/Stats/StatsFilterModalConnector.js delete mode 100644 src/NzbDrone.Core/IndexerStats/IndexerStatisticsRepository.cs diff --git a/frontend/src/Indexer/Stats/Stats.js b/frontend/src/Indexer/Stats/Stats.js index b3b89974d..b063d638c 100644 --- a/frontend/src/Indexer/Stats/Stats.js +++ b/frontend/src/Indexer/Stats/Stats.js @@ -7,8 +7,10 @@ import LoadingIndicator from 'Components/Loading/LoadingIndicator'; import PageContent from 'Components/Page/PageContent'; import PageContentBody from 'Components/Page/PageContentBody'; import PageToolbar from 'Components/Page/Toolbar/PageToolbar'; -import { kinds } from 'Helpers/Props'; +import PageToolbarSection from 'Components/Page/Toolbar/PageToolbarSection'; +import { align, kinds } from 'Helpers/Props'; import getErrorMessage from 'Utilities/Object/getErrorMessage'; +import StatsFilterMenu from './StatsFilterMenu'; import styles from './Stats.css'; function getAverageResponseTimeData(indexerStats) { @@ -144,14 +146,29 @@ function Stats(props) { item, isFetching, isPopulated, - error + error, + filters, + selectedFilterKey, + onFilterSelect } = props; const isLoaded = !!(!error && isPopulated); return ( - + + + + + { isFetching && !isPopulated && @@ -232,6 +249,10 @@ Stats.propTypes = { item: PropTypes.object.isRequired, isFetching: PropTypes.bool.isRequired, isPopulated: PropTypes.bool.isRequired, + filters: PropTypes.arrayOf(PropTypes.object).isRequired, + selectedFilterKey: PropTypes.string.isRequired, + customFilters: PropTypes.arrayOf(PropTypes.object).isRequired, + onFilterSelect: PropTypes.func.isRequired, error: PropTypes.object, data: PropTypes.object }; diff --git a/frontend/src/Indexer/Stats/StatsConnector.js b/frontend/src/Indexer/Stats/StatsConnector.js index 4dbc8fbd2..006716953 100644 --- a/frontend/src/Indexer/Stats/StatsConnector.js +++ b/frontend/src/Indexer/Stats/StatsConnector.js @@ -2,7 +2,7 @@ import PropTypes from 'prop-types'; import React, { Component } from 'react'; import { connect } from 'react-redux'; import { createSelector } from 'reselect'; -import { fetchIndexerStats } from 'Store/Actions/indexerStatsActions'; +import { fetchIndexerStats, setIndexerStatsFilter } from 'Store/Actions/indexerStatsActions'; import Stats from './Stats'; function createMapStateToProps() { @@ -12,9 +12,16 @@ function createMapStateToProps() { ); } -const mapDispatchToProps = { - dispatchFetchIndexers: fetchIndexerStats -}; +function createMapDispatchToProps(dispatch, props) { + return { + onFilterSelect(selectedFilterKey) { + dispatch(setIndexerStatsFilter({ selectedFilterKey })); + }, + dispatchFetchIndexerStats() { + dispatch(fetchIndexerStats()); + } + }; +} class StatsConnector extends Component { @@ -22,7 +29,7 @@ class StatsConnector extends Component { // Lifecycle componentDidMount() { - this.props.dispatchFetchIndexers(); + this.props.dispatchFetchIndexerStats(); } // @@ -38,7 +45,7 @@ class StatsConnector extends Component { } StatsConnector.propTypes = { - dispatchFetchIndexers: PropTypes.func.isRequired + dispatchFetchIndexerStats: PropTypes.func.isRequired }; -export default connect(createMapStateToProps, mapDispatchToProps)(StatsConnector); +export default connect(createMapStateToProps, createMapDispatchToProps)(StatsConnector); diff --git a/frontend/src/Indexer/Stats/StatsFilterMenu.js b/frontend/src/Indexer/Stats/StatsFilterMenu.js new file mode 100644 index 000000000..283159b7e --- /dev/null +++ b/frontend/src/Indexer/Stats/StatsFilterMenu.js @@ -0,0 +1,37 @@ +import PropTypes from 'prop-types'; +import React from 'react'; +import FilterMenu from 'Components/Menu/FilterMenu'; +import { align } from 'Helpers/Props'; + +function StatsFilterMenu(props) { + const { + selectedFilterKey, + filters, + isDisabled, + onFilterSelect + } = props; + + return ( + + ); +} + +StatsFilterMenu.propTypes = { + selectedFilterKey: PropTypes.oneOfType([PropTypes.string, PropTypes.number]).isRequired, + filters: PropTypes.arrayOf(PropTypes.object).isRequired, + isDisabled: PropTypes.bool.isRequired, + onFilterSelect: PropTypes.func.isRequired +}; + +StatsFilterMenu.defaultProps = { + showCustomFilters: false +}; + +export default StatsFilterMenu; diff --git a/frontend/src/Indexer/Stats/StatsFilterModalConnector.js b/frontend/src/Indexer/Stats/StatsFilterModalConnector.js new file mode 100644 index 000000000..53bf2ed3c --- /dev/null +++ b/frontend/src/Indexer/Stats/StatsFilterModalConnector.js @@ -0,0 +1,24 @@ +import { connect } from 'react-redux'; +import { createSelector } from 'reselect'; +import FilterModal from 'Components/Filter/FilterModal'; +import { setIndexerStatsFilter } from 'Store/Actions/indexerStatsActions'; + +function createMapStateToProps() { + return createSelector( + (state) => state.indexerStats.items, + (state) => state.indexerStats.filterBuilderProps, + (sectionItems, filterBuilderProps) => { + return { + sectionItems, + filterBuilderProps, + customFilterType: 'indexerStats' + }; + } + ); +} + +const mapDispatchToProps = { + dispatchSetFilter: setIndexerStatsFilter +}; + +export default connect(createMapStateToProps, mapDispatchToProps)(FilterModal); diff --git a/frontend/src/Store/Actions/indexerStatsActions.js b/frontend/src/Store/Actions/indexerStatsActions.js index ef2a7ccb0..e937cee93 100644 --- a/frontend/src/Store/Actions/indexerStatsActions.js +++ b/frontend/src/Store/Actions/indexerStatsActions.js @@ -1,5 +1,10 @@ +import moment from 'moment'; +import { batchActions } from 'redux-batched-actions'; +import { filterBuilderTypes, filterBuilderValueTypes } from 'Helpers/Props'; import { createThunk, handleThunks } from 'Store/thunks'; -import createFetchHandler from './Creators/createFetchHandler'; +import createAjaxRequest from 'Utilities/createAjaxRequest'; +import translate from 'Utilities/String/translate'; +import { set, update } from './baseActions'; import createHandleActions from './Creators/createHandleActions'; // @@ -15,30 +20,140 @@ export const defaultState = { isPopulated: false, error: null, item: {}, + start: null, + end: null, details: { isFetching: false, isPopulated: false, error: null, item: [] - } + }, + + filters: [ + { + key: 'all', + label: translate('All'), + filters: [] + }, + { + key: 'lastSeven', + label: 'Last 7 Days', + filters: [] + }, + { + key: 'lastThirty', + label: 'Last 30 Days', + filters: [] + }, + { + key: 'lastNinety', + label: 'Last 90 Days', + filters: [] + } + ], + + filterBuilderProps: [ + { + name: 'startDate', + label: 'Start Date', + type: filterBuilderTypes.EXACT, + valueType: filterBuilderValueTypes.DATE + }, + { + name: 'endDate', + label: 'End Date', + type: filterBuilderTypes.EXACT, + valueType: filterBuilderValueTypes.DATE + } + ], + selectedFilterKey: 'all' }; +export const persistState = [ + 'indexerStats.customFilters', + 'indexerStats.selectedFilterKey' +]; + // // Actions Types export const FETCH_INDEXER_STATS = 'indexerStats/fetchIndexerStats'; +export const SET_INDEXER_STATS_FILTER = 'indexerStats/setIndexerStatsFilter'; // // Action Creators export const fetchIndexerStats = createThunk(FETCH_INDEXER_STATS); +export const setIndexerStatsFilter = createThunk(SET_INDEXER_STATS_FILTER); // // Action Handlers export const actionHandlers = handleThunks({ - [FETCH_INDEXER_STATS]: createFetchHandler(section, '/indexerStats') + [FETCH_INDEXER_STATS]: function(getState, payload, dispatch) { + const state = getState(); + const indexerStats = state.indexerStats; + + const requestParams = { + endDate: moment().toISOString() + }; + + if (indexerStats.selectedFilterKey !== 'all') { + let dayCount = 7; + + if (indexerStats.selectedFilterKey === 'lastThirty') { + dayCount = 30; + } + + if (indexerStats.selectedFilterKey === 'lastNinety') { + dayCount = 90; + } + + requestParams.startDate = moment().add(-dayCount, 'days').endOf('day').toISOString(); + } + + const basesAttrs = { + section, + isFetching: true + }; + + const attrs = basesAttrs; + + dispatch(set(attrs)); + + const promise = createAjaxRequest({ + url: '/indexerStats', + data: requestParams + }).request; + + promise.done((data) => { + dispatch(batchActions([ + update({ section, data }), + + set({ + section, + isFetching: false, + isPopulated: true, + error: null + }) + ])); + }); + + promise.fail((xhr) => { + dispatch(set({ + section, + isFetching: false, + isPopulated: false, + error: xhr + })); + }); + }, + + [SET_INDEXER_STATS_FILTER]: function(getState, payload, dispatch) { + dispatch(set({ section, ...payload })); + dispatch(fetchIndexerStats()); + } }); // diff --git a/src/NzbDrone.Core/History/HistoryRepository.cs b/src/NzbDrone.Core/History/HistoryRepository.cs index 52bced1a7..e5e535fd8 100644 --- a/src/NzbDrone.Core/History/HistoryRepository.cs +++ b/src/NzbDrone.Core/History/HistoryRepository.cs @@ -15,6 +15,7 @@ namespace NzbDrone.Core.History List GetByIndexerId(int indexerId, HistoryEventType? eventType); void DeleteForIndexers(List indexerIds); History MostRecentForIndexer(int indexerId); + List Between(DateTime start, DateTime end); List Since(DateTime date, HistoryEventType? eventType); void Cleanup(int days); int CountSince(int indexerId, DateTime date, List eventTypes); @@ -78,6 +79,13 @@ namespace NzbDrone.Core.History .FirstOrDefault(); } + public List Between(DateTime start, DateTime end) + { + var builder = Builder().Where(x => x.Date >= start && x.Date <= end); + + return Query(builder).OrderBy(h => h.Date).ToList(); + } + public List Since(DateTime date, HistoryEventType? eventType) { var builder = Builder().Where(x => x.Date >= date); diff --git a/src/NzbDrone.Core/History/HistoryService.cs b/src/NzbDrone.Core/History/HistoryService.cs index 3187f1f20..685a36882 100644 --- a/src/NzbDrone.Core/History/HistoryService.cs +++ b/src/NzbDrone.Core/History/HistoryService.cs @@ -24,6 +24,7 @@ namespace NzbDrone.Core.History List FindByDownloadId(string downloadId); List GetByIndexerId(int indexerId, HistoryEventType? eventType); void UpdateMany(List toUpdate); + List Between(DateTime start, DateTime end); List Since(DateTime date, HistoryEventType? eventType); int CountSince(int indexerId, DateTime date, List eventTypes); } @@ -87,6 +88,11 @@ namespace NzbDrone.Core.History _historyRepository.UpdateMany(toUpdate); } + public List Between(DateTime start, DateTime end) + { + return _historyRepository.Between(start, end); + } + public List Since(DateTime date, HistoryEventType? eventType) { return _historyRepository.Since(date, eventType); diff --git a/src/NzbDrone.Core/IndexerStats/IndexerStatistics.cs b/src/NzbDrone.Core/IndexerStats/IndexerStatistics.cs index b1e58a722..261d0052a 100644 --- a/src/NzbDrone.Core/IndexerStats/IndexerStatistics.cs +++ b/src/NzbDrone.Core/IndexerStats/IndexerStatistics.cs @@ -1,7 +1,15 @@ +using System.Collections.Generic; using NzbDrone.Core.Datastore; namespace NzbDrone.Core.IndexerStats { + public class CombinedStatistics + { + public List IndexerStatistics { get; set; } + public List UserAgentStatistics { get; set; } + public List HostStatistics { get; set; } + } + public class IndexerStatistics : ResultSet { public int IndexerId { get; set; } diff --git a/src/NzbDrone.Core/IndexerStats/IndexerStatisticsRepository.cs b/src/NzbDrone.Core/IndexerStats/IndexerStatisticsRepository.cs deleted file mode 100644 index 0a11c0e16..000000000 --- a/src/NzbDrone.Core/IndexerStats/IndexerStatisticsRepository.cs +++ /dev/null @@ -1,103 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using Dapper; -using NzbDrone.Core.Datastore; -using NzbDrone.Core.Indexers; - -namespace NzbDrone.Core.IndexerStats -{ - public interface IIndexerStatisticsRepository - { - List IndexerStatistics(); - List UserAgentStatistics(); - List HostStatistics(); - } - - public class IndexerStatisticsRepository : IIndexerStatisticsRepository - { - private const string _selectTemplate = "SELECT /**select**/ FROM History /**join**/ /**innerjoin**/ /**leftjoin**/ /**where**/ /**groupby**/ /**having**/ /**orderby**/"; - - private readonly IMainDatabase _database; - - public IndexerStatisticsRepository(IMainDatabase database) - { - _database = database; - } - - public List IndexerStatistics() - { - var time = DateTime.UtcNow; - return Query(IndexerBuilder()); - } - - public List UserAgentStatistics() - { - var time = DateTime.UtcNow; - return UserAgentQuery(UserAgentBuilder()); - } - - public List HostStatistics() - { - var time = DateTime.UtcNow; - return HostQuery(HostBuilder()); - } - - private List Query(SqlBuilder builder) - { - var sql = builder.AddTemplate(_selectTemplate).LogQuery(); - - using (var conn = _database.OpenConnection()) - { - return conn.Query(sql.RawSql, sql.Parameters).ToList(); - } - } - - private List UserAgentQuery(SqlBuilder builder) - { - var sql = builder.AddTemplate(_selectTemplate).LogQuery(); - - using (var conn = _database.OpenConnection()) - { - return conn.Query(sql.RawSql, sql.Parameters).ToList(); - } - } - - private List HostQuery(SqlBuilder builder) - { - var sql = builder.AddTemplate(_selectTemplate).LogQuery(); - - using (var conn = _database.OpenConnection()) - { - return conn.Query(sql.RawSql, sql.Parameters).ToList(); - } - } - - private SqlBuilder IndexerBuilder() => new SqlBuilder() - .Select(@"Indexers.Id AS IndexerId, - Indexers.Name AS IndexerName, - SUM(CASE WHEN EventType == 2 then 1 else 0 end) AS NumberOfQueries, - SUM(CASE WHEN EventType == 2 AND Successful == 0 then 1 else 0 end) AS NumberOfFailedQueries, - SUM(CASE WHEN EventType == 3 then 1 else 0 end) AS NumberOfRssQueries, - SUM(CASE WHEN EventType == 3 AND Successful == 0 then 1 else 0 end) AS NumberOfFailedRssQueries, - SUM(CASE WHEN EventType == 4 then 1 else 0 end) AS NumberOfAuthQueries, - SUM(CASE WHEN EventType == 4 AND Successful == 0 then 1 else 0 end) AS NumberOfFailedAuthQueries, - SUM(CASE WHEN EventType == 1 then 1 else 0 end) AS NumberOfGrabs, - SUM(CASE WHEN EventType == 1 AND Successful == 0 then 1 else 0 end) AS NumberOfFailedGrabs, - AVG(json_extract(History.Data,'$.elapsedTime')) AS AverageResponseTime") - .Join((t, r) => t.IndexerId == r.Id) - .GroupBy(x => x.Id); - - private SqlBuilder UserAgentBuilder() => new SqlBuilder() - .Select(@"json_extract(History.Data,'$.source') AS UserAgent, - SUM(CASE WHEN EventType == 2 OR EventType == 3 then 1 else 0 end) AS NumberOfQueries, - SUM(CASE WHEN EventType == 1 then 1 else 0 end) AS NumberOfGrabs") - .GroupBy("UserAgent"); - - private SqlBuilder HostBuilder() => new SqlBuilder() - .Select(@"json_extract(History.Data,'$.host') AS Host, - SUM(CASE WHEN EventType == 2 OR EventType == 3 then 1 else 0 end) AS NumberOfQueries, - SUM(CASE WHEN EventType == 1 then 1 else 0 end) AS NumberOfGrabs") - .GroupBy("Host"); - } -} diff --git a/src/NzbDrone.Core/IndexerStats/IndexerStatisticsService.cs b/src/NzbDrone.Core/IndexerStats/IndexerStatisticsService.cs index 0e3eb370f..7bd8cd075 100644 --- a/src/NzbDrone.Core/IndexerStats/IndexerStatisticsService.cs +++ b/src/NzbDrone.Core/IndexerStats/IndexerStatisticsService.cs @@ -1,43 +1,174 @@ +using System; using System.Collections.Generic; using System.Linq; +using NzbDrone.Core.History; +using NzbDrone.Core.Indexers; namespace NzbDrone.Core.IndexerStats { public interface IIndexerStatisticsService { - List IndexerStatistics(); - List UserAgentStatistics(); - List HostStatistics(); + CombinedStatistics IndexerStatistics(DateTime start, DateTime end); } public class IndexerStatisticsService : IIndexerStatisticsService { - private readonly IIndexerStatisticsRepository _indexerStatisticsRepository; + private readonly IIndexerFactory _indexerFactory; + private readonly IHistoryService _historyService; - public IndexerStatisticsService(IIndexerStatisticsRepository indexerStatisticsRepository) + public IndexerStatisticsService(IHistoryService historyService, IIndexerFactory indexerFactory) { - _indexerStatisticsRepository = indexerStatisticsRepository; + _historyService = historyService; + _indexerFactory = indexerFactory; } - public List IndexerStatistics() + public CombinedStatistics IndexerStatistics(DateTime start, DateTime end) { - var indexerStatistics = _indexerStatisticsRepository.IndexerStatistics(); + var history = _historyService.Between(start, end); - return indexerStatistics.ToList(); - } + var groupedByIndexer = history.GroupBy(h => h.IndexerId); + var groupedByUserAgent = history.GroupBy(h => h.Data.GetValueOrDefault("source") ?? ""); + var groupedByHost = history.GroupBy(h => h.Data.GetValueOrDefault("host") ?? ""); - public List UserAgentStatistics() - { - var userAgentStatistics = _indexerStatisticsRepository.UserAgentStatistics(); + var indexerStatsList = new List(); + var userAgentStatsList = new List(); + var hostStatsList = new List(); - return userAgentStatistics.ToList(); - } + var indexers = _indexerFactory.All(); - public List HostStatistics() - { - var hostStatistics = _indexerStatisticsRepository.HostStatistics(); + foreach (var indexer in groupedByIndexer) + { + var indexerDef = indexers.SingleOrDefault(i => i.Id == indexer.Key); - return hostStatistics.ToList(); + if (indexerDef == null) + { + continue; + } + + var indexerStats = new IndexerStatistics + { + IndexerId = indexer.Key, + IndexerName = indexerDef.Name + }; + + var sortedEvents = indexer.OrderBy(v => v.Date) + .ThenBy(v => v.Id) + .ToArray(); + + indexerStats.AverageResponseTime = (int)sortedEvents.Where(h => h.Data.ContainsKey("elapsedTime")).Select(h => int.Parse(h.Data.GetValueOrDefault("elapsedTime"))).Average(); + + foreach (var historyEvent in sortedEvents) + { + var failed = !historyEvent.Successful; + switch (historyEvent.EventType) + { + case HistoryEventType.IndexerQuery: + indexerStats.NumberOfQueries++; + if (failed) + { + indexerStats.NumberOfFailedQueries++; + } + + break; + case HistoryEventType.IndexerAuth: + indexerStats.NumberOfAuthQueries++; + if (failed) + { + indexerStats.NumberOfFailedAuthQueries++; + } + + break; + case HistoryEventType.ReleaseGrabbed: + indexerStats.NumberOfGrabs++; + if (failed) + { + indexerStats.NumberOfFailedGrabs++; + } + + break; + case HistoryEventType.IndexerRss: + indexerStats.NumberOfRssQueries++; + if (failed) + { + indexerStats.NumberOfFailedRssQueries++; + } + + break; + default: + break; + } + } + + indexerStatsList.Add(indexerStats); + } + + foreach (var indexer in groupedByUserAgent) + { + var indexerStats = new UserAgentStatistics + { + UserAgent = indexer.Key + }; + + var sortedEvents = indexer.OrderBy(v => v.Date) + .ThenBy(v => v.Id) + .ToArray(); + + foreach (var historyEvent in sortedEvents) + { + switch (historyEvent.EventType) + { + case HistoryEventType.IndexerRss: + case HistoryEventType.IndexerQuery: + indexerStats.NumberOfQueries++; + + break; + case HistoryEventType.ReleaseGrabbed: + indexerStats.NumberOfGrabs++; + break; + default: + break; + } + } + + userAgentStatsList.Add(indexerStats); + } + + foreach (var indexer in groupedByHost) + { + var indexerStats = new HostStatistics + { + Host = indexer.Key + }; + + var sortedEvents = indexer.OrderBy(v => v.Date) + .ThenBy(v => v.Id) + .ToArray(); + + foreach (var historyEvent in sortedEvents) + { + switch (historyEvent.EventType) + { + case HistoryEventType.IndexerRss: + case HistoryEventType.IndexerQuery: + indexerStats.NumberOfQueries++; + break; + case HistoryEventType.ReleaseGrabbed: + indexerStats.NumberOfGrabs++; + break; + default: + break; + } + } + + hostStatsList.Add(indexerStats); + } + + return new CombinedStatistics + { + IndexerStatistics = indexerStatsList, + UserAgentStatistics = userAgentStatsList, + HostStatistics = hostStatsList + }; } } } diff --git a/src/Prowlarr.Api.V1/Indexers/IndexerStatsController.cs b/src/Prowlarr.Api.V1/Indexers/IndexerStatsController.cs index 10c5519f7..e3a13d066 100644 --- a/src/Prowlarr.Api.V1/Indexers/IndexerStatsController.cs +++ b/src/Prowlarr.Api.V1/Indexers/IndexerStatsController.cs @@ -1,3 +1,4 @@ +using System; using Microsoft.AspNetCore.Mvc; using NzbDrone.Core.IndexerStats; using Prowlarr.Http; @@ -15,13 +16,16 @@ namespace Prowlarr.Api.V1.Indexers } [HttpGet] - public IndexerStatsResource GetAll() + public IndexerStatsResource GetAll(DateTime? startDate, DateTime? endDate) { + var statsStartDate = startDate ?? DateTime.Now.AddDays(-30); + var statsEndDate = endDate ?? DateTime.Now; + var indexerResource = new IndexerStatsResource { - Indexers = _indexerStatisticsService.IndexerStatistics(), - UserAgents = _indexerStatisticsService.UserAgentStatistics(), - Hosts = _indexerStatisticsService.HostStatistics() + Indexers = _indexerStatisticsService.IndexerStatistics(statsStartDate, statsEndDate).IndexerStatistics, + UserAgents = _indexerStatisticsService.IndexerStatistics(statsStartDate, statsEndDate).UserAgentStatistics, + Hosts = _indexerStatisticsService.IndexerStatistics(statsStartDate, statsEndDate).HostStatistics }; return indexerResource;