mirror of
https://github.com/Jackett/Jackett.git
synced 2025-09-17 17:34:09 +02:00
internetarchive: C# -> yaml
This commit is contained in:
Garfield69
106
src/Jackett.Common/Definitions/internetarchive.yml
Normal file
106
src/Jackett.Common/Definitions/internetarchive.yml
Normal file
@@ -0,0 +1,106 @@
|
|||||||
|
---
|
||||||
|
id: internetarchive
|
||||||
|
name: Internet Archive
|
||||||
|
description: "Internet Archive is a non-profit digital library offering free universal access to books, movies & music, as well as 406 billion archived web pages"
|
||||||
|
language: en-US
|
||||||
|
type: public
|
||||||
|
encoding: UTF-8
|
||||||
|
links:
|
||||||
|
- https://archive.org/
|
||||||
|
|
||||||
|
caps:
|
||||||
|
categorymappings:
|
||||||
|
- {id: audio, cat: Audio, desc: audio}
|
||||||
|
- {id: etree, cat: Audio, desc: etree}
|
||||||
|
- {id: movies, cat: Movies, desc: movies}
|
||||||
|
- {id: movies, cat: TV, desc: movies}
|
||||||
|
- {id: image, cat: Other/Misc, desc: image}
|
||||||
|
- {id: texts, cat: Books, desc: texts}
|
||||||
|
- {id: software, cat: PC, desc: software}
|
||||||
|
- {id: web, cat: Other, desc: web}
|
||||||
|
- {id: collection, cat: Other, desc: collection}
|
||||||
|
- {id: account, cat: Other, desc: account}
|
||||||
|
- {id: data, cat: Other, desc: data}
|
||||||
|
- {id: other, cat: Other, desc: other}
|
||||||
|
|
||||||
|
modes:
|
||||||
|
search: [q]
|
||||||
|
tv-search: [q]
|
||||||
|
movie-search: [q]
|
||||||
|
music-search: [q]
|
||||||
|
book-search: [q]
|
||||||
|
|
||||||
|
settings:
|
||||||
|
- name: titleOnly
|
||||||
|
type: checkbox
|
||||||
|
label: Search only in title
|
||||||
|
default: true
|
||||||
|
- name: noMagnet
|
||||||
|
type: checkbox
|
||||||
|
label: Download using .torrent only. No Magnets.
|
||||||
|
default: false
|
||||||
|
- name: sort
|
||||||
|
type: select
|
||||||
|
label: Sort requested from site
|
||||||
|
default: publicdate
|
||||||
|
options:
|
||||||
|
publicdate: created
|
||||||
|
downloads: downloads
|
||||||
|
item_size: size
|
||||||
|
- name: type
|
||||||
|
type: select
|
||||||
|
label: Order requested from site
|
||||||
|
default: "-"
|
||||||
|
options:
|
||||||
|
"-": desc
|
||||||
|
"_": asc
|
||||||
|
|
||||||
|
search:
|
||||||
|
paths:
|
||||||
|
- path: advancedsearch.php
|
||||||
|
response:
|
||||||
|
type: json
|
||||||
|
inputs:
|
||||||
|
q: "{{ if and .Config.titleOnly .Keywords }}title:({{ else }}{{ end }}{{ if .Keywords }}{{ .Keywords }}{{ else }}{{ end }}{{ if and .Config.titleOnly .Keywords }}){{ else }}{{ end }}{{ if .Keywords }} AND {{ else }}{{ end }}format:(\"Archive BitTorrent\"){{ if .Categories }} AND mediatype:({{ join .Categories \" OR \" }}){{ else }}{{ end }}"
|
||||||
|
fl[]: "identifier,title,mediatype,item_size,downloads,btih,publicdate"
|
||||||
|
sort: "{{ if .Keywords }}{{ re_replace .Config.type \"_\" \"\" }}{{ .Config.sort }}{{ else }}-publicdate{{ end }}"
|
||||||
|
rows: 100
|
||||||
|
output: json
|
||||||
|
|
||||||
|
rows:
|
||||||
|
selector: response.docs
|
||||||
|
count:
|
||||||
|
selector: response.numFound
|
||||||
|
|
||||||
|
fields:
|
||||||
|
category:
|
||||||
|
selector: mediatype
|
||||||
|
title:
|
||||||
|
selector: title
|
||||||
|
id:
|
||||||
|
selector: identifier
|
||||||
|
details:
|
||||||
|
text: "details/{{ .Result.id }}"
|
||||||
|
download:
|
||||||
|
text: "download/{{ .Result.id }}/{{ .Result.id }}_archive.torrent"
|
||||||
|
btih:
|
||||||
|
selector: btih
|
||||||
|
optional: true
|
||||||
|
infohash:
|
||||||
|
text: "{{ if .Config.noMagnet }}{{ else }}{{ .Result.btih }}{{ end }}"
|
||||||
|
date:
|
||||||
|
# 2021-10-25T16:44:43Z
|
||||||
|
selector: publicdate
|
||||||
|
size:
|
||||||
|
selector: item_size
|
||||||
|
grabs:
|
||||||
|
selector: downloads
|
||||||
|
seeders:
|
||||||
|
text: 1
|
||||||
|
leechers:
|
||||||
|
text: 1
|
||||||
|
downloadvolumefactor:
|
||||||
|
text: 0
|
||||||
|
uploadvolumefactor:
|
||||||
|
text: 1
|
||||||
|
# json Elasticsearch
|
||||||
@@ -1,221 +0,0 @@
|
|||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Collections.Specialized;
|
|
||||||
using System.Diagnostics.CodeAnalysis;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Net;
|
|
||||||
using System.Text;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
using Jackett.Common.Models;
|
|
||||||
using Jackett.Common.Models.IndexerConfig;
|
|
||||||
using Jackett.Common.Services.Interfaces;
|
|
||||||
using Jackett.Common.Utils;
|
|
||||||
using Jackett.Common.Utils.Clients;
|
|
||||||
using Newtonsoft.Json;
|
|
||||||
using Newtonsoft.Json.Linq;
|
|
||||||
using NLog;
|
|
||||||
using static Jackett.Common.Models.IndexerConfig.ConfigurationData;
|
|
||||||
using WebClient = Jackett.Common.Utils.Clients.WebClient;
|
|
||||||
|
|
||||||
namespace Jackett.Common.Indexers
|
|
||||||
{
|
|
||||||
[ExcludeFromCodeCoverage]
|
|
||||||
public class InternetArchive : BaseWebIndexer
|
|
||||||
{
|
|
||||||
private string SearchUrl => SiteLink + "advancedsearch.php";
|
|
||||||
private string DetailsUrl => SiteLink + "details/";
|
|
||||||
private string LinkUrl => SiteLink + "download/";
|
|
||||||
|
|
||||||
private string _sort;
|
|
||||||
private string _order;
|
|
||||||
private bool _titleOnly;
|
|
||||||
private bool _noMagnet;
|
|
||||||
|
|
||||||
private ConfigurationData ConfigData => configData;
|
|
||||||
|
|
||||||
public InternetArchive(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps,
|
|
||||||
ICacheService cs)
|
|
||||||
: base(id: "internetarchive",
|
|
||||||
name: "Internet Archive",
|
|
||||||
description: "Internet Archive is a non-profit digital library offering free universal access to books, movies & music, as well as 406 billion archived web pages",
|
|
||||||
link: "https://archive.org/",
|
|
||||||
caps: new TorznabCapabilities
|
|
||||||
{
|
|
||||||
TvSearchParams = new List<TvSearchParam>
|
|
||||||
{
|
|
||||||
TvSearchParam.Q
|
|
||||||
},
|
|
||||||
MovieSearchParams = new List<MovieSearchParam>
|
|
||||||
{
|
|
||||||
MovieSearchParam.Q
|
|
||||||
},
|
|
||||||
MusicSearchParams = new List<MusicSearchParam>
|
|
||||||
{
|
|
||||||
MusicSearchParam.Q
|
|
||||||
},
|
|
||||||
BookSearchParams = new List<BookSearchParam>
|
|
||||||
{
|
|
||||||
BookSearchParam.Q
|
|
||||||
}
|
|
||||||
},
|
|
||||||
configService: configService,
|
|
||||||
client: wc,
|
|
||||||
logger: l,
|
|
||||||
p: ps,
|
|
||||||
cacheService: cs,
|
|
||||||
configData: new ConfigurationData())
|
|
||||||
{
|
|
||||||
Encoding = Encoding.UTF8;
|
|
||||||
Language = "en-US";
|
|
||||||
Type = "public";
|
|
||||||
|
|
||||||
var sort = new SingleSelectConfigurationItem("Sort requested from site", new Dictionary<string, string>
|
|
||||||
{
|
|
||||||
{"publicdate", "created"},
|
|
||||||
{"downloads", "downloads"},
|
|
||||||
{"item_size", "size"}
|
|
||||||
})
|
|
||||||
{ Value = "publicdate" };
|
|
||||||
configData.AddDynamic("sort", sort);
|
|
||||||
|
|
||||||
var order = new SingleSelectConfigurationItem("Order requested from site", new Dictionary<string, string>
|
|
||||||
{
|
|
||||||
{"desc", "desc"},
|
|
||||||
{"asc", "asc"}
|
|
||||||
})
|
|
||||||
{ Value = "desc" };
|
|
||||||
configData.AddDynamic("order", order);
|
|
||||||
|
|
||||||
var titleOnly = new BoolConfigurationItem("Search only in title") { Value = true };
|
|
||||||
configData.AddDynamic("titleOnly", titleOnly);
|
|
||||||
|
|
||||||
var noMagnet = new BoolConfigurationItem("Download using .torrent only. No Magnets.") { Value = false };
|
|
||||||
configData.AddDynamic("noMagnet", noMagnet);
|
|
||||||
|
|
||||||
AddCategoryMapping("audio", TorznabCatType.Audio);
|
|
||||||
AddCategoryMapping("etree", TorznabCatType.Audio);
|
|
||||||
AddCategoryMapping("movies", TorznabCatType.Movies);
|
|
||||||
AddCategoryMapping("movies", TorznabCatType.TV);
|
|
||||||
AddCategoryMapping("image", TorznabCatType.OtherMisc);
|
|
||||||
AddCategoryMapping("texts", TorznabCatType.Books);
|
|
||||||
AddCategoryMapping("software", TorznabCatType.PC);
|
|
||||||
AddCategoryMapping("web", TorznabCatType.Other);
|
|
||||||
AddCategoryMapping("collection", TorznabCatType.Other);
|
|
||||||
AddCategoryMapping("account", TorznabCatType.Other);
|
|
||||||
AddCategoryMapping("data", TorznabCatType.Other);
|
|
||||||
AddCategoryMapping("other", TorznabCatType.Other);
|
|
||||||
}
|
|
||||||
|
|
||||||
public override async Task<IndexerConfigurationStatus> ApplyConfiguration(JToken configJson)
|
|
||||||
{
|
|
||||||
LoadValuesFromJson(configJson);
|
|
||||||
var releases = await PerformQuery(new TorznabQuery());
|
|
||||||
|
|
||||||
await ConfigureIfOK(string.Empty, releases.Any(), () =>
|
|
||||||
throw new Exception("Could not find release from this URL."));
|
|
||||||
|
|
||||||
return IndexerConfigurationStatus.Completed;
|
|
||||||
}
|
|
||||||
|
|
||||||
public override void LoadValuesFromJson(JToken jsonConfig, bool useProtectionService = false)
|
|
||||||
{
|
|
||||||
base.LoadValuesFromJson(jsonConfig, useProtectionService);
|
|
||||||
|
|
||||||
var sort = (SingleSelectConfigurationItem)configData.GetDynamic("sort");
|
|
||||||
_sort = sort != null ? sort.Value : "publicdate";
|
|
||||||
|
|
||||||
var order = (SingleSelectConfigurationItem)configData.GetDynamic("order");
|
|
||||||
_order = order != null && order.Value.Equals("asc") ? "" : "-";
|
|
||||||
|
|
||||||
var titleOnly = (BoolConfigurationItem)configData.GetDynamic("titleOnly");
|
|
||||||
_titleOnly = titleOnly != null && titleOnly.Value;
|
|
||||||
|
|
||||||
var noMagnet = (BoolConfigurationItem)configData.GetDynamic("noMagnet");
|
|
||||||
_noMagnet = noMagnet != null && noMagnet.Value;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override async Task<IEnumerable<ReleaseInfo>> PerformQuery(TorznabQuery query)
|
|
||||||
{
|
|
||||||
var releases = new List<ReleaseInfo>();
|
|
||||||
|
|
||||||
var searchTerm = "format:(\"Archive BitTorrent\")";
|
|
||||||
var sort = "-publicdate";
|
|
||||||
if (!string.IsNullOrEmpty(query.SearchTerm))
|
|
||||||
{
|
|
||||||
if (_titleOnly)
|
|
||||||
searchTerm = "title:(" + query.SearchTerm + ") AND " + searchTerm;
|
|
||||||
else
|
|
||||||
searchTerm = query.SearchTerm + " AND " + searchTerm;
|
|
||||||
sort = _order + _sort;
|
|
||||||
}
|
|
||||||
|
|
||||||
var querycats = MapTorznabCapsToTrackers(query);
|
|
||||||
if (querycats.Any())
|
|
||||||
searchTerm += " AND mediatype:(" + string.Join(" OR ", querycats) + ")";
|
|
||||||
|
|
||||||
var qc = new NameValueCollection
|
|
||||||
{
|
|
||||||
{"q", searchTerm},
|
|
||||||
{"fl[]", "identifier,title,mediatype,item_size,downloads,btih,publicdate"},
|
|
||||||
{"sort", sort},
|
|
||||||
{"rows", "100"},
|
|
||||||
{"output", "json"}
|
|
||||||
};
|
|
||||||
var fullSearchUrl = SearchUrl + "?" + qc.GetQueryString();
|
|
||||||
var result = await RequestWithCookiesAndRetryAsync(fullSearchUrl);
|
|
||||||
foreach (var torrent in ParseResponse(result))
|
|
||||||
releases.Add(MakeRelease(torrent));
|
|
||||||
|
|
||||||
return releases;
|
|
||||||
}
|
|
||||||
|
|
||||||
private JArray ParseResponse(WebResult result)
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
if (result.Status != HttpStatusCode.OK)
|
|
||||||
throw new Exception("Response code error. HTTP code: " + result.Status);
|
|
||||||
var json = JsonConvert.DeserializeObject<dynamic>(result.ContentString);
|
|
||||||
if (!(json is JObject) || !(json["response"] is JObject) || !(json["response"]["docs"] is JArray))
|
|
||||||
throw new Exception("Response format error");
|
|
||||||
return (JArray)json["response"]["docs"];
|
|
||||||
}
|
|
||||||
catch (Exception e)
|
|
||||||
{
|
|
||||||
logger.Error("ParseResponse Error: ", e.Message);
|
|
||||||
throw new ExceptionWithConfigData(result.ContentString, ConfigData);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private ReleaseInfo MakeRelease(JToken torrent)
|
|
||||||
{
|
|
||||||
var id = GetFieldAs<string>("identifier", torrent);
|
|
||||||
var title = GetFieldAs<string>("title", torrent) ?? id;
|
|
||||||
var details = new Uri(DetailsUrl + id);
|
|
||||||
var btih = GetFieldAs<string>("btih", torrent);
|
|
||||||
var link = new Uri(LinkUrl + id + "/" + id + "_archive.torrent");
|
|
||||||
|
|
||||||
var release = new ReleaseInfo
|
|
||||||
{
|
|
||||||
Title = title,
|
|
||||||
Details = details,
|
|
||||||
Guid = details,
|
|
||||||
PublishDate = GetFieldAs<DateTime>("publicdate", torrent),
|
|
||||||
Category = MapTrackerCatToNewznab(GetFieldAs<string>("mediatype", torrent)),
|
|
||||||
Size = GetFieldAs<long>("item_size", torrent),
|
|
||||||
Seeders = 1,
|
|
||||||
Peers = 2,
|
|
||||||
Grabs = GetFieldAs<long>("downloads", torrent),
|
|
||||||
Link = link,
|
|
||||||
InfoHash = _noMagnet ? null : btih, // magnet link is auto generated from infohash
|
|
||||||
DownloadVolumeFactor = 0,
|
|
||||||
UploadVolumeFactor = 1
|
|
||||||
};
|
|
||||||
|
|
||||||
return release;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static T GetFieldAs<T>(string field, JToken torrent) =>
|
|
||||||
torrent[field] is JArray array ? array.First.ToObject<T>() : torrent.Value<T>(field);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user