feat(watchlist): Cache watchlist requests with matching E-Tags (#3901)

* perf(watchlist): add E-Tag caching to Plex watchlist requests

* refactor(watchlist): increase frequency of watchlist requests

* fix: sync watchlist every 3 min instead of 3 sec

server/api/plextv.ts

@@ -127,6 +127,11 @@ export interface PlexWatchlistItem {
   title: string;
 }
 
+export interface PlexWatchlistCache {
+  etag: string;
+  response: WatchlistResponse;
+}
+
 class PlexTvAPI extends ExternalAPI {
   private authToken: string;
 
@@ -270,6 +275,11 @@ class PlexTvAPI extends ExternalAPI {
     items: PlexWatchlistItem[];
   }> {
     try {
+      const watchlistCache = cacheManager.getCache('plexwatchlist');
+      let cachedWatchlist = watchlistCache.data.get<PlexWatchlistCache>(
+        this.authToken
+      );
+
       const response = await this.axios.get<WatchlistResponse>(
         '/library/sections/watchlist/all',
         {
@@ -277,12 +287,29 @@ class PlexTvAPI extends ExternalAPI {
             'X-Plex-Container-Start': offset,
             'X-Plex-Container-Size': size,
           },
+          headers: {
+            'If-None-Match': cachedWatchlist?.etag,
+          },
           baseURL: 'https://metadata.provider.plex.tv',
+          validateStatus: (status) => status < 400, // Allow HTTP 304 to return without error
         }
       );
 
+      // If we don't recieve HTTP 304, the watchlist has been updated and we need to update the cache.
+      if (response.status >= 200 && response.status <= 299) {
+        cachedWatchlist = {
+          etag: response.headers.etag,
+          response: response.data,
+        };
+
+        watchlistCache.data.set<PlexWatchlistCache>(
+          this.authToken,
+          cachedWatchlist
+        );
+      }
+
       const watchlistDetails = await Promise.all(
-        (response.data.MediaContainer.Metadata ?? []).map(
+        (cachedWatchlist?.response.MediaContainer.Metadata ?? []).map(
           async (watchlistItem) => {
             const detailedResponse = await this.getRolling<MetadataResponse>(
               `/library/metadata/${watchlistItem.ratingKey}`,
@@ -320,7 +347,7 @@ class PlexTvAPI extends ExternalAPI {
       return {
         offset,
         size,
-        totalSize: response.data.MediaContainer.totalSize,
+        totalSize: cachedWatchlist?.response.MediaContainer.totalSize ?? 0,
         items: filteredList,
       };
     } catch (e) {

server/job/schedule.ts

@@ -8,7 +8,6 @@ import type { JobId } from '@server/lib/settings';
 import { getSettings } from '@server/lib/settings';
 import watchlistSync from '@server/lib/watchlistsync';
 import logger from '@server/logger';
-import random from 'lodash/random';
 import schedule from 'node-schedule';
 
 interface ScheduledJob {
@@ -62,30 +61,20 @@ export const startJobs = (): void => {
   });
 
   // Watchlist Sync
-  const watchlistSyncJob: ScheduledJob = {
+  scheduledJobs.push({
     id: 'plex-watchlist-sync',
     name: 'Plex Watchlist Sync',
     type: 'process',
-    interval: 'fixed',
+    interval: 'seconds',
     cronSchedule: jobs['plex-watchlist-sync'].schedule,
-    job: schedule.scheduleJob(new Date(Date.now() + 1000 * 60 * 20), () => {
+    job: schedule.scheduleJob(jobs['plex-watchlist-sync'].schedule, () => {
       logger.info('Starting scheduled job: Plex Watchlist Sync', {
         label: 'Jobs',
       });
       watchlistSync.syncWatchlist();
     }),
-  };
-
-  // To help alleviate load on Plex's servers, we will add some fuzziness to the next schedule
-  // after each run
-  watchlistSyncJob.job.on('run', () => {
-    watchlistSyncJob.job.schedule(
-      new Date(Math.floor(Date.now() + 1000 * 60 * random(14, 24, true)))
-    );
   });
 
-  scheduledJobs.push(watchlistSyncJob);
-
   // Run full radarr scan every 24 hours
   scheduledJobs.push({
     id: 'radarr-scan',

server/lib/cache.ts

@@ -8,7 +8,8 @@ export type AvailableCacheIds =
   | 'imdb'
   | 'github'
   | 'plexguid'
-  | 'plextv';
+  | 'plextv'
+  | 'plexwatchlist';
 
 const DEFAULT_TTL = 300;
 const DEFAULT_CHECK_PERIOD = 120;
@@ -68,6 +69,7 @@ class CacheManager {
       stdTtl: 86400 * 7, // 1 week cache
       checkPeriod: 60,
     }),
+    plexwatchlist: new Cache('plexwatchlist', 'Plex Watchlist'),
   };
 
   public getCache(id: AvailableCacheIds): Cache {

server/lib/settings.ts

@@ -407,7 +407,7 @@ class Settings {
           schedule: '0 0 3 * * *',
         },
         'plex-watchlist-sync': {
-          schedule: '0 */10 * * * *',
+          schedule: '0 */3 * * * *',
         },
         'radarr-scan': {
           schedule: '0 0 4 * * *',

server/lib/watchlistsync.ts

@@ -62,7 +62,7 @@ class WatchlistSync {
 
     const plexTvApi = new PlexTvAPI(user.plexToken);
 
-    const response = await plexTvApi.getWatchlist({ size: 200 });
+    const response = await plexTvApi.getWatchlist({ size: 20 });
 
     const mediaItems = await Media.getRelatedMedia(
       response.items.map((i) => i.tmdbId)