feat(plex-scan): plex scanner improvements (#2105)

server/api/plexapi.ts

@@ -1,5 +1,5 @@
 import NodePlexAPI from 'plex-api';
-import { getSettings, PlexSettings } from '../lib/settings';
+import { getSettings, Library, PlexSettings } from '../lib/settings';
 
 export interface PlexLibraryItem {
   ratingKey: string;
@@ -11,11 +11,16 @@ export interface PlexLibraryItem {
   grandparentGuid?: string;
   addedAt: number;
   updatedAt: number;
+  Guid?: {
+    id: string;
+  }[];
   type: 'movie' | 'show' | 'season' | 'episode';
+  Media: Media[];
 }
 
 interface PlexLibraryResponse {
   MediaContainer: {
+    totalSize: number;
     Metadata: PlexLibraryItem[];
   };
 }
@@ -137,12 +142,50 @@ class PlexAPI {
     return response.MediaContainer.Directory;
   }
 
-  public async getLibraryContents(id: string): Promise<PlexLibraryItem[]> {
-    const response = await this.plexClient.query<PlexLibraryResponse>(
-      `/library/sections/${id}/all`
-    );
+  public async syncLibraries(): Promise<void> {
+    const settings = getSettings();
 
-    return response.MediaContainer.Metadata ?? [];
+    const libraries = await this.getLibraries();
+
+    const newLibraries: Library[] = libraries
+      // Remove libraries that are not movie or show
+      .filter((library) => library.type === 'movie' || library.type === 'show')
+      // Remove libraries that do not have a metadata agent set (usually personal video libraries)
+      .filter((library) => library.agent !== 'com.plexapp.agents.none')
+      .map((library) => {
+        const existing = settings.plex.libraries.find(
+          (l) => l.id === library.key && l.name === library.title
+        );
+
+        return {
+          id: library.key,
+          name: library.title,
+          enabled: existing?.enabled ?? false,
+          type: library.type,
+          lastScan: existing?.lastScan,
+        };
+      });
+
+    settings.plex.libraries = newLibraries;
+    settings.save();
+  }
+
+  public async getLibraryContents(
+    id: string,
+    { offset = 0, size = 50 }: { offset?: number; size?: number } = {}
+  ): Promise<{ totalSize: number; items: PlexLibraryItem[] }> {
+    const response = await this.plexClient.query<PlexLibraryResponse>({
+      uri: `/library/sections/${id}/all?includeGuids=1`,
+      extraHeaders: {
+        'X-Plex-Container-Start': `${offset}`,
+        'X-Plex-Container-Size': `${size}`,
+      },
+    });
+
+    return {
+      totalSize: response.MediaContainer.totalSize,
+      items: response.MediaContainer.Metadata ?? [],
+    };
   }
 
   public async getMetadata(
@@ -166,10 +209,17 @@ class PlexAPI {
     return response.MediaContainer.Metadata;
   }
 
-  public async getRecentlyAdded(id: string): Promise<PlexLibraryItem[]> {
-    const response = await this.plexClient.query<PlexLibraryResponse>(
-      `/library/sections/${id}/recentlyAdded`
-    );
+  public async getRecentlyAdded(
+    id: string,
+    options: { addedAt: number } = {
+      addedAt: Date.now() - 1000 * 60 * 60,
+    }
+  ): Promise<PlexLibraryItem[]> {
+    const response = await this.plexClient.query<PlexLibraryResponse>({
+      uri: `/library/sections/${id}/all?sort=addedAt%3Adesc&addedAt>>=${Math.floor(
+        options.addedAt / 1000
+      )}`,
+    });
 
     return response.MediaContainer.Metadata;
   }

server/index.ts

@@ -10,7 +10,9 @@ import path from 'path';
 import swaggerUi from 'swagger-ui-express';
 import { createConnection, getRepository } from 'typeorm';
 import YAML from 'yamljs';
+import PlexAPI from './api/plexapi';
 import { Session } from './entity/Session';
+import { User } from './entity/User';
 import { startJobs } from './job/schedule';
 import notificationManager from './lib/notifications';
 import DiscordAgent from './lib/notifications/agents/discord';
@@ -49,6 +51,26 @@ app
     // Load Settings
     const settings = getSettings().load();
 
+    // Migrate library types
+    if (
+      settings.plex.libraries.length > 1 &&
+      !settings.plex.libraries[0].type
+    ) {
+      const userRepository = getRepository(User);
+      const admin = await userRepository.findOne({
+        select: ['id', 'plexToken'],
+        order: { id: 'ASC' },
+      });
+
+      if (admin) {
+        const plexapi = new PlexAPI({ plexToken: admin.plexToken });
+        await plexapi.syncLibraries();
+        logger.info('Migrating libraries to include media type', {
+          label: 'Settings',
+        });
+      }
+    }
+
     // Register Notification Agents
     notificationManager.registerAgents([
       new DiscordAgent(),

server/lib/cache.ts

@@ -1,6 +1,12 @@
 import NodeCache from 'node-cache';
 
-export type AvailableCacheIds = 'tmdb' | 'radarr' | 'sonarr' | 'rt' | 'github';
+export type AvailableCacheIds =
+  | 'tmdb'
+  | 'radarr'
+  | 'sonarr'
+  | 'rt'
+  | 'github'
+  | 'plexguid';
 
 const DEFAULT_TTL = 300;
 const DEFAULT_CHECK_PERIOD = 120;
@@ -48,6 +54,10 @@ class CacheManager {
       stdTtl: 21600,
       checkPeriod: 60 * 30,
     }),
+    plexguid: new Cache('plexguid', 'Plex GUID Cache', {
+      stdTtl: 86400 * 7, // 1 week cache
+      checkPeriod: 60 * 30,
+    }),
   };
 
   public getCache(id: AvailableCacheIds): Cache {

server/lib/scanners/baseScanner.ts

@@ -55,6 +55,7 @@ class BaseScanner<T> {
   private updateRate;
   protected progress = 0;
   protected items: T[] = [];
+  protected totalSize?: number = 0;
   protected scannerName: string;
   protected enable4kMovie = false;
   protected enable4kShow = false;
@@ -609,6 +610,14 @@ class BaseScanner<T> {
   ): void {
     logger[level](message, { label: this.scannerName, ...optional });
   }
+
+  get protectedUpdateRate(): number {
+    return this.updateRate;
+  }
+
+  get protectedBundleSize(): number {
+    return this.bundleSize;
+  }
 }
 
 export default BaseScanner;

server/lib/scanners/plex/index.ts

@@ -4,6 +4,7 @@ import animeList from '../../../api/animelist';
 import PlexAPI, { PlexLibraryItem, PlexMetadata } from '../../../api/plexapi';
 import { TmdbTvDetails } from '../../../api/themoviedb/interfaces';
 import { User } from '../../../entity/User';
+import cacheManager from '../../cache';
 import { getSettings, Library } from '../../settings';
 import BaseScanner, {
   MediaIds,
@@ -38,7 +39,7 @@ class PlexScanner
   private isRecentOnly = false;
 
   public constructor(isRecentOnly = false) {
-    super('Plex Scan');
+    super('Plex Scan', { bundleSize: 50 });
     this.isRecentOnly = isRecentOnly;
   }
 
@@ -46,7 +47,7 @@ class PlexScanner
     return {
       running: this.running,
       progress: this.progress,
-      total: this.items.length,
+      total: this.totalSize ?? 0,
       currentLibrary: this.currentLibrary,
       libraries: this.libraries,
     };
@@ -82,10 +83,17 @@ class PlexScanner
           this.currentLibrary = library;
           this.log(
             `Beginning to process recently added for library: ${library.name}`,
-            'info'
+            'info',
+            { lastScan: library.lastScan }
           );
           const libraryItems = await this.plexClient.getRecentlyAdded(
-            library.id
+            library.id,
+            library.lastScan
+              ? {
+                  // We remove 10 minutes from the last scan as a buffer
+                  addedAt: library.lastScan - 1000 * 60 * 10,
+                }
+              : undefined
           );
 
           // Bundle items up by rating keys
@@ -104,13 +112,26 @@ class PlexScanner
           });
 
           await this.loop(this.processItem.bind(this), { sessionId });
+
+          // After run completes, update last scan time
+          const newLibraries = settings.plex.libraries.map((lib) => {
+            if (lib.id === library.id) {
+              return {
+                ...lib,
+                lastScan: Date.now(),
+              };
+            }
+            return lib;
+          });
+
+          settings.plex.libraries = newLibraries;
+          settings.save();
         }
       } else {
         for (const library of this.libraries) {
           this.currentLibrary = library;
           this.log(`Beginning to process library: ${library.name}`, 'info');
-          this.items = await this.plexClient.getLibraryContents(library.id);
-          await this.loop(this.processItem.bind(this), { sessionId });
+          await this.paginateLibrary(library, { sessionId });
         }
       }
       this.log(
@@ -126,6 +147,52 @@ class PlexScanner
     }
   }
 
+  private async paginateLibrary(
+    library: Library,
+    { start = 0, sessionId }: { start?: number; sessionId: string }
+  ) {
+    if (!this.running) {
+      throw new Error('Sync was aborted.');
+    }
+
+    if (this.sessionId !== sessionId) {
+      throw new Error('New session was started. Old session aborted.');
+    }
+
+    const response = await this.plexClient.getLibraryContents(library.id, {
+      size: this.protectedBundleSize,
+      offset: start,
+    });
+
+    this.progress = start;
+    this.totalSize = response.totalSize;
+
+    if (response.items.length === 0) {
+      return;
+    }
+
+    await Promise.all(
+      response.items.map(async (item) => {
+        await this.processItem(item);
+      })
+    );
+
+    if (response.items.length < this.protectedBundleSize) {
+      return;
+    }
+
+    await new Promise<void>((resolve, reject) =>
+      setTimeout(() => {
+        this.paginateLibrary(library, {
+          start: start + this.protectedBundleSize,
+          sessionId,
+        })
+          .then(() => resolve())
+          .catch((e) => reject(new Error(e.message)));
+      }, this.protectedUpdateRate)
+    );
+  }
+
   private async processItem(plexitem: PlexLibraryItem) {
     try {
       if (plexitem.type === 'movie') {
@@ -147,9 +214,8 @@ class PlexScanner
 
   private async processPlexMovie(plexitem: PlexLibraryItem) {
     const mediaIds = await this.getMediaIds(plexitem);
-    const metadata = await this.plexClient.getMetadata(plexitem.ratingKey);
 
-    const has4k = metadata.Media.some(
+    const has4k = plexitem.Media.some(
       (media) => media.videoResolution === '4k'
     );
 
@@ -263,10 +329,25 @@ class PlexScanner
   }
 
   private async getMediaIds(plexitem: PlexLibraryItem): Promise<MediaIds> {
-    const mediaIds: Partial<MediaIds> = {};
+    let mediaIds: Partial<MediaIds> = {};
     // Check if item is using new plex movie/tv agent
     if (plexitem.guid.match(plexRegex)) {
-      const metadata = await this.plexClient.getMetadata(plexitem.ratingKey);
+      const guidCache = cacheManager.getCache('plexguid');
+
+      const cachedGuids = guidCache.data.get<MediaIds>(plexitem.ratingKey);
+
+      if (cachedGuids) {
+        this.log('GUIDs are cached. Skipping metadata request.', 'debug', {
+          mediaIds: cachedGuids,
+          title: plexitem.title,
+        });
+        mediaIds = cachedGuids;
+      }
+
+      const metadata =
+        plexitem.Guid && plexitem.Guid.length > 0
+          ? plexitem
+          : await this.plexClient.getMetadata(plexitem.ratingKey);
 
       // If there is no Guid field at all, then we bail
       if (!metadata.Guid) {
@@ -295,6 +376,10 @@ class PlexScanner
         });
         mediaIds.tmdbId = tmdbMovie.id;
       }
+
+      // Cache GUIDs
+      guidCache.data.set(plexitem.ratingKey, mediaIds);
+
       // Check if the agent is IMDb
     } else if (plexitem.guid.match(imdbRegex)) {
       const imdbMatch = plexitem.guid.match(imdbRegex);

server/lib/settings.ts

@@ -9,6 +9,8 @@ export interface Library {
   id: string;
   name: string;
   enabled: boolean;
+  type: 'show' | 'movie';
+  lastScan?: number;
 }
 
 export interface Region {

server/routes/settings/index.ts

@@ -20,7 +20,7 @@ import { scheduledJobs } from '../../job/schedule';
 import cacheManager, { AvailableCacheIds } from '../../lib/cache';
 import { Permission } from '../../lib/permissions';
 import { plexFullScanner } from '../../lib/scanners/plex';
-import { getSettings, Library, MainSettings } from '../../lib/settings';
+import { getSettings, MainSettings } from '../../lib/settings';
 import logger from '../../logger';
 import { isAuthenticated } from '../../middleware/auth';
 import { getAppVersion } from '../../utils/appVersion';
@@ -197,26 +197,7 @@ settingsRoutes.get('/plex/library', async (req, res) => {
     });
     const plexapi = new PlexAPI({ plexToken: admin.plexToken });
 
-    const libraries = await plexapi.getLibraries();
-
-    const newLibraries: Library[] = libraries
-      // Remove libraries that are not movie or show
-      .filter((library) => library.type === 'movie' || library.type === 'show')
-      // Remove libraries that do not have a metadata agent set (usually personal video libraries)
-      .filter((library) => library.agent !== 'com.plexapp.agents.none')
-      .map((library) => {
-        const existing = settings.plex.libraries.find(
-          (l) => l.id === library.key && l.name === library.title
-        );
-
-        return {
-          id: library.key,
-          name: library.title,
-          enabled: existing?.enabled ?? false,
-        };
-      });
-
-    settings.plex.libraries = newLibraries;
+    await plexapi.syncLibraries();
   }
 
   const enabledLibraries = req.query.enable

server/types/plex-api.d.ts

@@ -21,6 +21,13 @@ declare module 'plex-api' {
       requestOptions?: Record<string, string | number>;
     });
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    query: <T extends Record<string, any>>(endpoint: string) => Promise<T>;
+    query: <T extends Record<string, any>>(
+      endpoint:
+        | string
+        | {
+            uri: string;
+            extraHeaders?: Record<string, string | number>;
+          }
+    ) => Promise<T>;
   }
 }