diff --git a/src/Jackett.Common/Definitions/internetarchive.yml b/src/Jackett.Common/Definitions/internetarchive.yml new file mode 100644 index 000000000..de925c99e --- /dev/null +++ b/src/Jackett.Common/Definitions/internetarchive.yml @@ -0,0 +1,106 @@ +--- +id: internetarchive +name: Internet Archive +description: "Internet Archive is a non-profit digital library offering free universal access to books, movies & music, as well as 406 billion archived web pages" +language: en-US +type: public +encoding: UTF-8 +links: + - https://archive.org/ + +caps: + categorymappings: + - {id: audio, cat: Audio, desc: audio} + - {id: etree, cat: Audio, desc: etree} + - {id: movies, cat: Movies, desc: movies} + - {id: movies, cat: TV, desc: movies} + - {id: image, cat: Other/Misc, desc: image} + - {id: texts, cat: Books, desc: texts} + - {id: software, cat: PC, desc: software} + - {id: web, cat: Other, desc: web} + - {id: collection, cat: Other, desc: collection} + - {id: account, cat: Other, desc: account} + - {id: data, cat: Other, desc: data} + - {id: other, cat: Other, desc: other} + + modes: + search: [q] + tv-search: [q] + movie-search: [q] + music-search: [q] + book-search: [q] + +settings: + - name: titleOnly + type: checkbox + label: Search only in title + default: true + - name: noMagnet + type: checkbox + label: Download using .torrent only. No Magnets. + default: false + - name: sort + type: select + label: Sort requested from site + default: publicdate + options: + publicdate: created + downloads: downloads + item_size: size + - name: type + type: select + label: Order requested from site + default: "-" + options: + "-": desc + "_": asc + +search: + paths: + - path: advancedsearch.php + response: + type: json + inputs: + q: "{{ if and .Config.titleOnly .Keywords }}title:({{ else }}{{ end }}{{ if .Keywords }}{{ .Keywords }}{{ else }}{{ end }}{{ if and .Config.titleOnly .Keywords }}){{ else }}{{ end }}{{ if .Keywords }} AND {{ else }}{{ end }}format:(\"Archive BitTorrent\"){{ if .Categories }} AND mediatype:({{ join .Categories \" OR \" }}){{ else }}{{ end }}" + fl[]: "identifier,title,mediatype,item_size,downloads,btih,publicdate" + sort: "{{ if .Keywords }}{{ re_replace .Config.type \"_\" \"\" }}{{ .Config.sort }}{{ else }}-publicdate{{ end }}" + rows: 100 + output: json + + rows: + selector: response.docs + count: + selector: response.numFound + + fields: + category: + selector: mediatype + title: + selector: title + id: + selector: identifier + details: + text: "details/{{ .Result.id }}" + download: + text: "download/{{ .Result.id }}/{{ .Result.id }}_archive.torrent" + btih: + selector: btih + optional: true + infohash: + text: "{{ if .Config.noMagnet }}{{ else }}{{ .Result.btih }}{{ end }}" + date: + # 2021-10-25T16:44:43Z + selector: publicdate + size: + selector: item_size + grabs: + selector: downloads + seeders: + text: 1 + leechers: + text: 1 + downloadvolumefactor: + text: 0 + uploadvolumefactor: + text: 1 +# json Elasticsearch diff --git a/src/Jackett.Common/Indexers/InternetArchive.cs b/src/Jackett.Common/Indexers/InternetArchive.cs deleted file mode 100644 index bcb7b583f..000000000 --- a/src/Jackett.Common/Indexers/InternetArchive.cs +++ /dev/null @@ -1,221 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Collections.Specialized; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Net; -using System.Text; -using System.Threading.Tasks; -using Jackett.Common.Models; -using Jackett.Common.Models.IndexerConfig; -using Jackett.Common.Services.Interfaces; -using Jackett.Common.Utils; -using Jackett.Common.Utils.Clients; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using NLog; -using static Jackett.Common.Models.IndexerConfig.ConfigurationData; -using WebClient = Jackett.Common.Utils.Clients.WebClient; - -namespace Jackett.Common.Indexers -{ - [ExcludeFromCodeCoverage] - public class InternetArchive : BaseWebIndexer - { - private string SearchUrl => SiteLink + "advancedsearch.php"; - private string DetailsUrl => SiteLink + "details/"; - private string LinkUrl => SiteLink + "download/"; - - private string _sort; - private string _order; - private bool _titleOnly; - private bool _noMagnet; - - private ConfigurationData ConfigData => configData; - - public InternetArchive(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps, - ICacheService cs) - : base(id: "internetarchive", - name: "Internet Archive", - description: "Internet Archive is a non-profit digital library offering free universal access to books, movies & music, as well as 406 billion archived web pages", - link: "https://archive.org/", - caps: new TorznabCapabilities - { - TvSearchParams = new List - { - TvSearchParam.Q - }, - MovieSearchParams = new List - { - MovieSearchParam.Q - }, - MusicSearchParams = new List - { - MusicSearchParam.Q - }, - BookSearchParams = new List - { - BookSearchParam.Q - } - }, - configService: configService, - client: wc, - logger: l, - p: ps, - cacheService: cs, - configData: new ConfigurationData()) - { - Encoding = Encoding.UTF8; - Language = "en-US"; - Type = "public"; - - var sort = new SingleSelectConfigurationItem("Sort requested from site", new Dictionary - { - {"publicdate", "created"}, - {"downloads", "downloads"}, - {"item_size", "size"} - }) - { Value = "publicdate" }; - configData.AddDynamic("sort", sort); - - var order = new SingleSelectConfigurationItem("Order requested from site", new Dictionary - { - {"desc", "desc"}, - {"asc", "asc"} - }) - { Value = "desc" }; - configData.AddDynamic("order", order); - - var titleOnly = new BoolConfigurationItem("Search only in title") { Value = true }; - configData.AddDynamic("titleOnly", titleOnly); - - var noMagnet = new BoolConfigurationItem("Download using .torrent only. No Magnets.") { Value = false }; - configData.AddDynamic("noMagnet", noMagnet); - - AddCategoryMapping("audio", TorznabCatType.Audio); - AddCategoryMapping("etree", TorznabCatType.Audio); - AddCategoryMapping("movies", TorznabCatType.Movies); - AddCategoryMapping("movies", TorznabCatType.TV); - AddCategoryMapping("image", TorznabCatType.OtherMisc); - AddCategoryMapping("texts", TorznabCatType.Books); - AddCategoryMapping("software", TorznabCatType.PC); - AddCategoryMapping("web", TorznabCatType.Other); - AddCategoryMapping("collection", TorznabCatType.Other); - AddCategoryMapping("account", TorznabCatType.Other); - AddCategoryMapping("data", TorznabCatType.Other); - AddCategoryMapping("other", TorznabCatType.Other); - } - - public override async Task ApplyConfiguration(JToken configJson) - { - LoadValuesFromJson(configJson); - var releases = await PerformQuery(new TorznabQuery()); - - await ConfigureIfOK(string.Empty, releases.Any(), () => - throw new Exception("Could not find release from this URL.")); - - return IndexerConfigurationStatus.Completed; - } - - public override void LoadValuesFromJson(JToken jsonConfig, bool useProtectionService = false) - { - base.LoadValuesFromJson(jsonConfig, useProtectionService); - - var sort = (SingleSelectConfigurationItem)configData.GetDynamic("sort"); - _sort = sort != null ? sort.Value : "publicdate"; - - var order = (SingleSelectConfigurationItem)configData.GetDynamic("order"); - _order = order != null && order.Value.Equals("asc") ? "" : "-"; - - var titleOnly = (BoolConfigurationItem)configData.GetDynamic("titleOnly"); - _titleOnly = titleOnly != null && titleOnly.Value; - - var noMagnet = (BoolConfigurationItem)configData.GetDynamic("noMagnet"); - _noMagnet = noMagnet != null && noMagnet.Value; - } - - protected override async Task> PerformQuery(TorznabQuery query) - { - var releases = new List(); - - var searchTerm = "format:(\"Archive BitTorrent\")"; - var sort = "-publicdate"; - if (!string.IsNullOrEmpty(query.SearchTerm)) - { - if (_titleOnly) - searchTerm = "title:(" + query.SearchTerm + ") AND " + searchTerm; - else - searchTerm = query.SearchTerm + " AND " + searchTerm; - sort = _order + _sort; - } - - var querycats = MapTorznabCapsToTrackers(query); - if (querycats.Any()) - searchTerm += " AND mediatype:(" + string.Join(" OR ", querycats) + ")"; - - var qc = new NameValueCollection - { - {"q", searchTerm}, - {"fl[]", "identifier,title,mediatype,item_size,downloads,btih,publicdate"}, - {"sort", sort}, - {"rows", "100"}, - {"output", "json"} - }; - var fullSearchUrl = SearchUrl + "?" + qc.GetQueryString(); - var result = await RequestWithCookiesAndRetryAsync(fullSearchUrl); - foreach (var torrent in ParseResponse(result)) - releases.Add(MakeRelease(torrent)); - - return releases; - } - - private JArray ParseResponse(WebResult result) - { - try - { - if (result.Status != HttpStatusCode.OK) - throw new Exception("Response code error. HTTP code: " + result.Status); - var json = JsonConvert.DeserializeObject(result.ContentString); - if (!(json is JObject) || !(json["response"] is JObject) || !(json["response"]["docs"] is JArray)) - throw new Exception("Response format error"); - return (JArray)json["response"]["docs"]; - } - catch (Exception e) - { - logger.Error("ParseResponse Error: ", e.Message); - throw new ExceptionWithConfigData(result.ContentString, ConfigData); - } - } - - private ReleaseInfo MakeRelease(JToken torrent) - { - var id = GetFieldAs("identifier", torrent); - var title = GetFieldAs("title", torrent) ?? id; - var details = new Uri(DetailsUrl + id); - var btih = GetFieldAs("btih", torrent); - var link = new Uri(LinkUrl + id + "/" + id + "_archive.torrent"); - - var release = new ReleaseInfo - { - Title = title, - Details = details, - Guid = details, - PublishDate = GetFieldAs("publicdate", torrent), - Category = MapTrackerCatToNewznab(GetFieldAs("mediatype", torrent)), - Size = GetFieldAs("item_size", torrent), - Seeders = 1, - Peers = 2, - Grabs = GetFieldAs("downloads", torrent), - Link = link, - InfoHash = _noMagnet ? null : btih, // magnet link is auto generated from infohash - DownloadVolumeFactor = 0, - UploadVolumeFactor = 1 - }; - - return release; - } - - private static T GetFieldAs(string field, JToken torrent) => - torrent[field] is JArray array ? array.First.ToObject() : torrent.Value(field); - } -}