mirror of
https://github.com/sct/overseerr.git
synced 2025-09-17 17:24:35 +02:00
feat: map AniDB IDs from Hama agent to tvdb/tmdb/imdb IDs (#538)
* feat: map AniDB IDs from Hama agent to tvdb/tmdb/imdb IDs re #453 * refactor: removes sync job for AnimeList, load mapping on demand * refactor: addressing review comments, using typescript types for xml parsing * refactor: make sure sync job does not update create same tvshow/movie twice Hama agent can have same tvdbid it for different library items - for example when user stores different seasons for same tv show separately. This change adds "AsyncLock" that guarantees code in callback runs for same id fully, before running same callback next time. * refactor: do not use season 0 tvdbid for tvshow from mapping file * refactor: support multiple imdb mappings for same anidb entry * refactor: add debug log for missing tvdb entries in tmdb lookups from anidb/hama agent
This commit is contained in:
Mārtiņš Možeiko
@@ -1,6 +1,6 @@
|
||||
import { getRepository } from 'typeorm';
|
||||
import { User } from '../../entity/User';
|
||||
import PlexAPI, { PlexLibraryItem } from '../../api/plexapi';
|
||||
import PlexAPI, { PlexLibraryItem, PlexMetadata } from '../../api/plexapi';
|
||||
import TheMovieDb, {
|
||||
TmdbMovieDetails,
|
||||
TmdbTvDetails,
|
||||
@@ -11,15 +11,22 @@ import logger from '../../logger';
|
||||
import { getSettings, Library } from '../../lib/settings';
|
||||
import Season from '../../entity/Season';
|
||||
import { uniqWith } from 'lodash';
|
||||
import animeList from '../../api/animelist';
|
||||
import AsyncLock from '../../utils/asyncLock';
|
||||
|
||||
const BUNDLE_SIZE = 20;
|
||||
const UPDATE_RATE = 4 * 1000;
|
||||
|
||||
const imdbRegex = new RegExp(/imdb:\/\/(tt[0-9]+)/);
|
||||
const tmdbRegex = new RegExp(/tmdb:\/\/([0-9]+)/);
|
||||
const tvdbRegex = new RegExp(/tvdb:\/\/([0-9]+)|hama:\/\/tvdb-([0-9]+)/);
|
||||
const tvdbRegex = new RegExp(/tvdb:\/\/([0-9]+)/);
|
||||
const tmdbShowRegex = new RegExp(/themoviedb:\/\/([0-9]+)/);
|
||||
const plexRegex = new RegExp(/plex:\/\//);
|
||||
// Hama agent uses ASS naming, see details here:
|
||||
// https://github.com/ZeroQI/Absolute-Series-Scanner/blob/master/README.md#forcing-the-movieseries-id
|
||||
const hamaTvdbRegex = new RegExp(/hama:\/\/tvdb[0-9]?-([0-9]+)/);
|
||||
const hamaAnidbRegex = new RegExp(/hama:\/\/anidb[0-9]?-([0-9]+)/);
|
||||
const HAMA_AGENT = 'com.plexapp.agents.hama';
|
||||
|
||||
interface SyncStatus {
|
||||
running: boolean;
|
||||
@@ -38,6 +45,7 @@ class JobPlexSync {
|
||||
private currentLibrary: Library;
|
||||
private running = false;
|
||||
private isRecentOnly = false;
|
||||
private asyncLock = new AsyncLock();
|
||||
|
||||
constructor({ isRecentOnly }: { isRecentOnly?: boolean } = {}) {
|
||||
this.tmdb = new TheMovieDb();
|
||||
@@ -78,26 +86,28 @@ class JobPlexSync {
|
||||
}
|
||||
});
|
||||
|
||||
const existing = await this.getExisting(
|
||||
newMedia.tmdbId,
|
||||
MediaType.MOVIE
|
||||
);
|
||||
|
||||
if (existing && existing.status === MediaStatus.AVAILABLE) {
|
||||
this.log(`Title exists and is already available ${metadata.title}`);
|
||||
} else if (existing && existing.status !== MediaStatus.AVAILABLE) {
|
||||
existing.status = MediaStatus.AVAILABLE;
|
||||
mediaRepository.save(existing);
|
||||
this.log(
|
||||
`Request for ${metadata.title} exists. Setting status AVAILABLE`,
|
||||
'info'
|
||||
await this.asyncLock.dispatch(newMedia.tmdbId, async () => {
|
||||
const existing = await this.getExisting(
|
||||
newMedia.tmdbId,
|
||||
MediaType.MOVIE
|
||||
);
|
||||
} else {
|
||||
newMedia.status = MediaStatus.AVAILABLE;
|
||||
newMedia.mediaType = MediaType.MOVIE;
|
||||
await mediaRepository.save(newMedia);
|
||||
this.log(`Saved ${plexitem.title}`);
|
||||
}
|
||||
|
||||
if (existing && existing.status === MediaStatus.AVAILABLE) {
|
||||
this.log(`Title exists and is already available ${metadata.title}`);
|
||||
} else if (existing && existing.status !== MediaStatus.AVAILABLE) {
|
||||
existing.status = MediaStatus.AVAILABLE;
|
||||
mediaRepository.save(existing);
|
||||
this.log(
|
||||
`Request for ${metadata.title} exists. Setting status AVAILABLE`,
|
||||
'info'
|
||||
);
|
||||
} else {
|
||||
newMedia.status = MediaStatus.AVAILABLE;
|
||||
newMedia.mediaType = MediaType.MOVIE;
|
||||
await mediaRepository.save(newMedia);
|
||||
this.log(`Saved ${plexitem.title}`);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
let tmdbMovieId: number | undefined;
|
||||
let tmdbMovie: TmdbMovieDetails | undefined;
|
||||
@@ -118,30 +128,7 @@ class JobPlexSync {
|
||||
throw new Error('Unable to find TMDB ID');
|
||||
}
|
||||
|
||||
const existing = await this.getExisting(tmdbMovieId, MediaType.MOVIE);
|
||||
if (existing && existing.status === MediaStatus.AVAILABLE) {
|
||||
this.log(`Title exists and is already available ${plexitem.title}`);
|
||||
} else if (existing && existing.status !== MediaStatus.AVAILABLE) {
|
||||
existing.status = MediaStatus.AVAILABLE;
|
||||
await mediaRepository.save(existing);
|
||||
this.log(
|
||||
`Request for ${plexitem.title} exists. Setting status AVAILABLE`,
|
||||
'info'
|
||||
);
|
||||
} else {
|
||||
// If we have a tmdb movie guid but it didn't already exist, only then
|
||||
// do we request the movie from tmdb (to reduce api requests)
|
||||
if (!tmdbMovie) {
|
||||
tmdbMovie = await this.tmdb.getMovie({ movieId: tmdbMovieId });
|
||||
}
|
||||
const newMedia = new Media();
|
||||
newMedia.imdbId = tmdbMovie.external_ids.imdb_id;
|
||||
newMedia.tmdbId = tmdbMovie.id;
|
||||
newMedia.status = MediaStatus.AVAILABLE;
|
||||
newMedia.mediaType = MediaType.MOVIE;
|
||||
await mediaRepository.save(newMedia);
|
||||
this.log(`Saved ${tmdbMovie.title}`);
|
||||
}
|
||||
await this.processMovieWithId(plexitem, tmdbMovie, tmdbMovieId);
|
||||
}
|
||||
} catch (e) {
|
||||
this.log(
|
||||
@@ -155,6 +142,71 @@ class JobPlexSync {
|
||||
}
|
||||
}
|
||||
|
||||
private async processMovieWithId(
|
||||
plexitem: PlexLibraryItem,
|
||||
tmdbMovie: TmdbMovieDetails | undefined,
|
||||
tmdbMovieId: number
|
||||
) {
|
||||
const mediaRepository = getRepository(Media);
|
||||
|
||||
await this.asyncLock.dispatch(tmdbMovieId, async () => {
|
||||
const existing = await this.getExisting(tmdbMovieId, MediaType.MOVIE);
|
||||
if (existing && existing.status === MediaStatus.AVAILABLE) {
|
||||
this.log(`Title exists and is already available ${plexitem.title}`);
|
||||
} else if (existing && existing.status !== MediaStatus.AVAILABLE) {
|
||||
existing.status = MediaStatus.AVAILABLE;
|
||||
await mediaRepository.save(existing);
|
||||
this.log(
|
||||
`Request for ${plexitem.title} exists. Setting status AVAILABLE`,
|
||||
'info'
|
||||
);
|
||||
} else {
|
||||
// If we have a tmdb movie guid but it didn't already exist, only then
|
||||
// do we request the movie from tmdb (to reduce api requests)
|
||||
if (!tmdbMovie) {
|
||||
tmdbMovie = await this.tmdb.getMovie({ movieId: tmdbMovieId });
|
||||
}
|
||||
const newMedia = new Media();
|
||||
newMedia.imdbId = tmdbMovie.external_ids.imdb_id;
|
||||
newMedia.tmdbId = tmdbMovie.id;
|
||||
newMedia.status = MediaStatus.AVAILABLE;
|
||||
newMedia.mediaType = MediaType.MOVIE;
|
||||
await mediaRepository.save(newMedia);
|
||||
this.log(`Saved ${tmdbMovie.title}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// this adds all movie episodes from specials season for Hama agent
|
||||
private async processHamaSpecials(metadata: PlexMetadata, tvdbId: number) {
|
||||
const specials = metadata.Children?.Metadata.find(
|
||||
(md) => Number(md.index) === 0
|
||||
);
|
||||
if (specials) {
|
||||
const episodes = await this.plexClient.getChildrenMetadata(
|
||||
specials.ratingKey
|
||||
);
|
||||
if (episodes) {
|
||||
for (const episode of episodes) {
|
||||
const special = await animeList.getSpecialEpisode(
|
||||
tvdbId,
|
||||
episode.index
|
||||
);
|
||||
if (special) {
|
||||
if (special.tmdbId) {
|
||||
await this.processMovieWithId(episode, undefined, special.tmdbId);
|
||||
} else if (special.imdbId) {
|
||||
const tmdbMovie = await this.tmdb.getMovieByImdbId({
|
||||
imdbId: special.imdbId,
|
||||
});
|
||||
await this.processMovieWithId(episode, tmdbMovie, tmdbMovie.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async processShow(plexitem: PlexLibraryItem) {
|
||||
const mediaRepository = getRepository(Media);
|
||||
|
||||
@@ -182,108 +234,182 @@ class JobPlexSync {
|
||||
if (matchedtmdb?.[1]) {
|
||||
tvShow = await this.tmdb.getTvShow({ tvId: Number(matchedtmdb[1]) });
|
||||
}
|
||||
}
|
||||
} else if (metadata.guid.match(hamaTvdbRegex)) {
|
||||
const matched = metadata.guid.match(hamaTvdbRegex);
|
||||
const tvdbId = matched?.[1];
|
||||
|
||||
if (tvShow && metadata) {
|
||||
// Lets get the available seasons from plex
|
||||
const seasons = tvShow.seasons;
|
||||
const media = await this.getExisting(tvShow.id, MediaType.TV);
|
||||
if (tvdbId) {
|
||||
tvShow = await this.tmdb.getShowByTvdbId({ tvdbId: Number(tvdbId) });
|
||||
if (animeList.isLoaded()) {
|
||||
await this.processHamaSpecials(metadata, Number(tvdbId));
|
||||
} else {
|
||||
this.log(
|
||||
`Hama id ${plexitem.guid} detected, but library agent is not set to Hama`,
|
||||
'warn'
|
||||
);
|
||||
}
|
||||
}
|
||||
} else if (metadata.guid.match(hamaAnidbRegex)) {
|
||||
const matched = metadata.guid.match(hamaAnidbRegex);
|
||||
|
||||
const newSeasons: Season[] = [];
|
||||
|
||||
const currentSeasonAvailable = (
|
||||
media?.seasons.filter(
|
||||
(season) => season.status === MediaStatus.AVAILABLE
|
||||
) ?? []
|
||||
).length;
|
||||
|
||||
seasons.forEach((season) => {
|
||||
const matchedPlexSeason = metadata.Children?.Metadata.find(
|
||||
(md) => Number(md.index) === season.season_number
|
||||
if (!animeList.isLoaded()) {
|
||||
this.log(
|
||||
`Hama id ${plexitem.guid} detected, but library agent is not set to Hama`,
|
||||
'warn'
|
||||
);
|
||||
} else if (matched?.[1]) {
|
||||
const anidbId = Number(matched[1]);
|
||||
const result = animeList.getFromAnidbId(anidbId);
|
||||
|
||||
const existingSeason = media?.seasons.find(
|
||||
(es) => es.seasonNumber === season.season_number
|
||||
);
|
||||
|
||||
// Check if we found the matching season and it has all the available episodes
|
||||
if (
|
||||
matchedPlexSeason &&
|
||||
Number(matchedPlexSeason.leafCount) === season.episode_count
|
||||
) {
|
||||
if (existingSeason) {
|
||||
existingSeason.status = MediaStatus.AVAILABLE;
|
||||
// first try to lookup tvshow by tvdbid
|
||||
if (result?.tvdbId) {
|
||||
const extResponse = await this.tmdb.getByExternalId({
|
||||
externalId: result.tvdbId,
|
||||
type: 'tvdb',
|
||||
});
|
||||
if (extResponse.tv_results[0]) {
|
||||
tvShow = await this.tmdb.getTvShow({
|
||||
tvId: extResponse.tv_results[0].id,
|
||||
});
|
||||
} else {
|
||||
newSeasons.push(
|
||||
new Season({
|
||||
seasonNumber: season.season_number,
|
||||
status: MediaStatus.AVAILABLE,
|
||||
})
|
||||
this.log(
|
||||
`Missing TVDB ${result.tvdbId} entry in TMDB for AniDB ${anidbId}`
|
||||
);
|
||||
}
|
||||
} else if (matchedPlexSeason) {
|
||||
if (existingSeason) {
|
||||
existingSeason.status = MediaStatus.PARTIALLY_AVAILABLE;
|
||||
} else {
|
||||
newSeasons.push(
|
||||
new Season({
|
||||
seasonNumber: season.season_number,
|
||||
status: MediaStatus.PARTIALLY_AVAILABLE,
|
||||
})
|
||||
await this.processHamaSpecials(metadata, result.tvdbId);
|
||||
}
|
||||
|
||||
if (!tvShow) {
|
||||
// if lookup of tvshow above failed, then try movie with tmdbid/imdbid
|
||||
// note - some tv shows have imdbid set too, that's why this need to go second
|
||||
if (result?.tmdbId) {
|
||||
return await this.processMovieWithId(
|
||||
plexitem,
|
||||
undefined,
|
||||
result.tmdbId
|
||||
);
|
||||
} else if (result?.imdbId) {
|
||||
const tmdbMovie = await this.tmdb.getMovieByImdbId({
|
||||
imdbId: result.imdbId,
|
||||
});
|
||||
return await this.processMovieWithId(
|
||||
plexitem,
|
||||
tmdbMovie,
|
||||
tmdbMovie.id
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Remove extras season. We dont count it for determining availability
|
||||
const filteredSeasons = tvShow.seasons.filter(
|
||||
(season) => season.season_number !== 0
|
||||
);
|
||||
if (tvShow) {
|
||||
await this.asyncLock.dispatch(tvShow.id, async () => {
|
||||
if (!tvShow) {
|
||||
// this will never execute, but typescript thinks somebody could reset tvShow from
|
||||
// outer scope back to null before this async gets called
|
||||
return;
|
||||
}
|
||||
|
||||
const isAllSeasons =
|
||||
newSeasons.length + (media?.seasons.length ?? 0) >=
|
||||
filteredSeasons.length;
|
||||
// Lets get the available seasons from plex
|
||||
const seasons = tvShow.seasons;
|
||||
const media = await this.getExisting(tvShow.id, MediaType.TV);
|
||||
|
||||
if (media) {
|
||||
// Update existing
|
||||
media.seasons = [...media.seasons, ...newSeasons];
|
||||
const newSeasons: Season[] = [];
|
||||
|
||||
const newSeasonAvailable = (
|
||||
media.seasons.filter(
|
||||
const currentSeasonAvailable = (
|
||||
media?.seasons.filter(
|
||||
(season) => season.status === MediaStatus.AVAILABLE
|
||||
) ?? []
|
||||
).length;
|
||||
|
||||
// If at least one new season has become available, update
|
||||
// the lastSeasonChange field so we can trigger notifications
|
||||
if (newSeasonAvailable > currentSeasonAvailable) {
|
||||
this.log(
|
||||
`Detected ${
|
||||
newSeasonAvailable - currentSeasonAvailable
|
||||
} new season(s) for ${tvShow.name}`,
|
||||
'debug'
|
||||
seasons.forEach((season) => {
|
||||
const matchedPlexSeason = metadata.Children?.Metadata.find(
|
||||
(md) => Number(md.index) === season.season_number
|
||||
);
|
||||
media.lastSeasonChange = new Date();
|
||||
}
|
||||
|
||||
media.status = isAllSeasons
|
||||
? MediaStatus.AVAILABLE
|
||||
: MediaStatus.PARTIALLY_AVAILABLE;
|
||||
await mediaRepository.save(media);
|
||||
this.log(`Updating existing title: ${tvShow.name}`);
|
||||
} else {
|
||||
const newMedia = new Media({
|
||||
mediaType: MediaType.TV,
|
||||
seasons: newSeasons,
|
||||
tmdbId: tvShow.id,
|
||||
tvdbId: tvShow.external_ids.tvdb_id,
|
||||
status: isAllSeasons
|
||||
? MediaStatus.AVAILABLE
|
||||
: MediaStatus.PARTIALLY_AVAILABLE,
|
||||
const existingSeason = media?.seasons.find(
|
||||
(es) => es.seasonNumber === season.season_number
|
||||
);
|
||||
|
||||
// Check if we found the matching season and it has all the available episodes
|
||||
if (
|
||||
matchedPlexSeason &&
|
||||
Number(matchedPlexSeason.leafCount) === season.episode_count
|
||||
) {
|
||||
if (existingSeason) {
|
||||
existingSeason.status = MediaStatus.AVAILABLE;
|
||||
} else {
|
||||
newSeasons.push(
|
||||
new Season({
|
||||
seasonNumber: season.season_number,
|
||||
status: MediaStatus.AVAILABLE,
|
||||
})
|
||||
);
|
||||
}
|
||||
} else if (matchedPlexSeason) {
|
||||
if (existingSeason) {
|
||||
existingSeason.status = MediaStatus.PARTIALLY_AVAILABLE;
|
||||
} else {
|
||||
newSeasons.push(
|
||||
new Season({
|
||||
seasonNumber: season.season_number,
|
||||
status: MediaStatus.PARTIALLY_AVAILABLE,
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
await mediaRepository.save(newMedia);
|
||||
this.log(`Saved ${tvShow.name}`);
|
||||
}
|
||||
|
||||
// Remove extras season. We dont count it for determining availability
|
||||
const filteredSeasons = tvShow.seasons.filter(
|
||||
(season) => season.season_number !== 0
|
||||
);
|
||||
|
||||
const isAllSeasons =
|
||||
newSeasons.length + (media?.seasons.length ?? 0) >=
|
||||
filteredSeasons.length;
|
||||
|
||||
if (media) {
|
||||
// Update existing
|
||||
media.seasons = [...media.seasons, ...newSeasons];
|
||||
|
||||
const newSeasonAvailable = (
|
||||
media.seasons.filter(
|
||||
(season) => season.status === MediaStatus.AVAILABLE
|
||||
) ?? []
|
||||
).length;
|
||||
|
||||
// If at least one new season has become available, update
|
||||
// the lastSeasonChange field so we can trigger notifications
|
||||
if (newSeasonAvailable > currentSeasonAvailable) {
|
||||
this.log(
|
||||
`Detected ${
|
||||
newSeasonAvailable - currentSeasonAvailable
|
||||
} new season(s) for ${tvShow.name}`,
|
||||
'debug'
|
||||
);
|
||||
media.lastSeasonChange = new Date();
|
||||
}
|
||||
|
||||
media.status = isAllSeasons
|
||||
? MediaStatus.AVAILABLE
|
||||
: MediaStatus.PARTIALLY_AVAILABLE;
|
||||
await mediaRepository.save(media);
|
||||
this.log(`Updating existing title: ${tvShow.name}`);
|
||||
} else {
|
||||
const newMedia = new Media({
|
||||
mediaType: MediaType.TV,
|
||||
seasons: newSeasons,
|
||||
tmdbId: tvShow.id,
|
||||
tvdbId: tvShow.external_ids.tvdb_id,
|
||||
status: isAllSeasons
|
||||
? MediaStatus.AVAILABLE
|
||||
: MediaStatus.PARTIALLY_AVAILABLE,
|
||||
});
|
||||
await mediaRepository.save(newMedia);
|
||||
this.log(`Saved ${tvShow.name}`);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
this.log(`failed show: ${plexitem.guid}`);
|
||||
}
|
||||
@@ -351,6 +477,17 @@ class JobPlexSync {
|
||||
logger[level](message, { label: 'Plex Sync', ...optional });
|
||||
}
|
||||
|
||||
// checks if any of this.libraries has Hama agent set in Plex
|
||||
private async hasHamaAgent() {
|
||||
const plexLibraries = await this.plexClient.getLibraries();
|
||||
return this.libraries.some((library) =>
|
||||
plexLibraries.some(
|
||||
(plexLibrary) =>
|
||||
plexLibrary.agent === HAMA_AGENT && library.id === plexLibrary.key
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
public async run(): Promise<void> {
|
||||
const settings = getSettings();
|
||||
if (!this.running) {
|
||||
@@ -371,6 +508,11 @@ class JobPlexSync {
|
||||
(library) => library.enabled
|
||||
);
|
||||
|
||||
const hasHama = await this.hasHamaAgent();
|
||||
if (hasHama) {
|
||||
await animeList.sync();
|
||||
}
|
||||
|
||||
if (this.isRecentOnly) {
|
||||
for (const library of this.libraries) {
|
||||
this.currentLibrary = library;
|
||||
|
||||
Reference in New Issue
Block a user