using System; using System.Collections.Generic; using System.Net; using NLog; using NzbDrone.Common; using NzbDrone.Common.Http; using NzbDrone.Core.Indexers.Exceptions; using NzbDrone.Core.IndexerSearch.Definitions; using NzbDrone.Core.Parser.Model; using System.Linq; namespace NzbDrone.Core.Indexers { public interface IFetchFeedFromIndexers { IList FetchRss(IIndexer indexer); IList Fetch(IIndexer indexer, SeasonSearchCriteria searchCriteria); IList Fetch(IIndexer indexer, SingleEpisodeSearchCriteria searchCriteria); IList Fetch(IIndexer indexer, DailyEpisodeSearchCriteria searchCriteria); IList Fetch(IIndexer indexer, AnimeEpisodeSearchCriteria searchCriteria); IList Fetch(IIndexer indexer, SpecialEpisodeSearchCriteria searchCriteria); } public class FetchFeedService : IFetchFeedFromIndexers { private readonly Logger _logger; private readonly IHttpProvider _httpProvider; public FetchFeedService(IHttpProvider httpProvider, Logger logger) { _httpProvider = httpProvider; _logger = logger; } public virtual IList FetchRss(IIndexer indexer) { _logger.Debug("Fetching feeds from " + indexer); var result = Fetch(indexer, indexer.RecentFeed); _logger.Debug("Finished processing feeds from {0} found {1} releases", indexer, result.Count); return result; } public IList Fetch(IIndexer indexer, SeasonSearchCriteria searchCriteria) { return Fetch(indexer, searchCriteria, 0).DistinctBy(c => c.DownloadUrl).ToList(); } private IList Fetch(IIndexer indexer, SeasonSearchCriteria searchCriteria, int offset) { var searchUrls = indexer.GetSeasonSearchUrls(searchCriteria.QueryTitles, searchCriteria.Series.TvRageId, searchCriteria.SeasonNumber, offset).ToList(); if (searchUrls.Any()) { _logger.Debug("Searching for {0} offset: {1}", searchCriteria, offset); var result = Fetch(indexer, searchUrls); _logger.Info("{0} offset {1}. Found {2}", indexer, offset, result.Count); if (indexer.SupportsPaging && result.Count >= indexer.SupportedPageSize && offset < 900) { result.AddRange(Fetch(indexer, searchCriteria, offset + indexer.SupportedPageSize)); } //Only log finish for the first call to this recursive method if (offset == 0) { _logger.Info("Finished searching {0} for {1}. Found {2}", indexer, searchCriteria, result.Count); } return result; } return new List(); } public IList Fetch(IIndexer indexer, SingleEpisodeSearchCriteria searchCriteria) { var searchUrls = indexer.GetEpisodeSearchUrls(searchCriteria.QueryTitles, searchCriteria.Series.TvRageId, searchCriteria.SeasonNumber, searchCriteria.EpisodeNumber).ToList(); return Fetch(indexer, searchUrls, searchCriteria); } public IList Fetch(IIndexer indexer, DailyEpisodeSearchCriteria searchCriteria) { var searchUrls = indexer.GetDailyEpisodeSearchUrls(searchCriteria.QueryTitles, searchCriteria.Series.TvRageId, searchCriteria.AirDate).ToList(); return Fetch(indexer, searchUrls, searchCriteria); } public IList Fetch(IIndexer indexer, AnimeEpisodeSearchCriteria searchCriteria) { var searchUrls = indexer.GetAnimeEpisodeSearchUrls(searchCriteria.SceneTitles, searchCriteria.Series.TvRageId, searchCriteria.AbsoluteEpisodeNumber).ToList(); return Fetch(indexer, searchUrls, searchCriteria); } public IList Fetch(IIndexer indexer, SpecialEpisodeSearchCriteria searchCriteria) { var searchUrls = new List(); foreach (var episodeQueryTitle in searchCriteria.EpisodeQueryTitles) { var urls = indexer.GetSearchUrls(episodeQueryTitle).ToList(); if (urls.Any()) { _logger.Debug("Performing query of {0} for {1}", indexer, episodeQueryTitle); searchUrls.AddRange(urls); } } return Fetch(indexer, searchUrls, searchCriteria); } private List Fetch(IIndexer indexer, IEnumerable urls, SearchCriteriaBase searchCriteria) { var urlList = urls.ToList(); if (urlList.Empty()) { return new List(); } _logger.Debug("Searching for {0}", searchCriteria); var result = Fetch(indexer, urlList); _logger.Info("Finished searching {0} for {1}. Found {2}", indexer, searchCriteria, result.Count); return result; } private List Fetch(IIndexer indexer, IEnumerable urls) { var result = new List(); foreach (var url in urls) { try { _logger.Debug("Downloading Feed " + url); var xml = _httpProvider.DownloadString(url); if (!string.IsNullOrWhiteSpace(xml)) { result.AddRange(indexer.Parser.Process(xml, url)); } else { _logger.Warn("{0} returned empty response.", url); } } catch (WebException webException) { if (webException.Message.Contains("502") || webException.Message.Contains("503") || webException.Message.Contains("timed out")) { _logger.Warn("{0} server is currently unavailable. {1} {2}", indexer, url, webException.Message); } else { _logger.Warn("{0} {1} {2}", indexer, url, webException.Message); } } catch (ApiKeyException) { _logger.Warn("Invalid API Key for {0} {1}", indexer, url); } catch (Exception feedEx) { feedEx.Data.Add("FeedUrl", url); _logger.ErrorException("An error occurred while processing feed. " + url, feedEx); } } result = result.DistinctBy(v => v.Guid).ToList(); result.ForEach(c => { c.Indexer = indexer.Definition.Name; c.DownloadProtocol = indexer.Protocol; }); return result; } } }