diff --git a/server/lib/imageproxy.ts b/server/lib/imageproxy.ts
index badfe94f2..04e320a0b 100644
--- a/server/lib/imageproxy.ts
+++ b/server/lib/imageproxy.ts
@@ -135,6 +135,7 @@ class ImageProxy {
   private cacheVersion;
   private key;
   private baseUrl;
+  private headers: HeadersInit | null = null;
 
   constructor(
     key: string,
@@ -142,6 +143,7 @@ class ImageProxy {
     options: {
       cacheVersion?: number;
       rateLimitOptions?: RateLimitOptions;
+      headers?: HeadersInit;
     } = {}
   ) {
     this.cacheVersion = options.cacheVersion ?? 1;
@@ -155,9 +157,13 @@ class ImageProxy {
     } else {
       this.fetch = fetch;
     }
+    this.headers = options.headers || null;
   }
 
-  public async getImage(path: string): Promise<ImageResponse> {
+  public async getImage(
+    path: string,
+    fallbackPath?: string
+  ): Promise<ImageResponse> {
     const cacheKey = this.getCacheKey(path);
 
     const imageResponse = await this.get(cacheKey);
@@ -166,7 +172,11 @@ class ImageProxy {
       const newImage = await this.set(path, cacheKey);
 
       if (!newImage) {
-        throw new Error('Failed to load image');
+        if (fallbackPath) {
+          return await this.getImage(fallbackPath);
+        } else {
+          throw new Error('Failed to load image');
+        }
       }
 
       return newImage;
@@ -247,7 +257,12 @@ class ImageProxy {
             : '/'
           : '') +
         (path.startsWith('/') ? path.slice(1) : path);
-      const response = await this.fetch(href);
+      const response = await this.fetch(href, {
+        headers: this.headers || undefined,
+      });
+      if (!response.ok) {
+        return null;
+      }
       const arrayBuffer = await response.arrayBuffer();
       const buffer = Buffer.from(arrayBuffer);
 
diff --git a/server/routes/auth.ts b/server/routes/auth.ts
index 560f04d57..70e674f97 100644
--- a/server/routes/auth.ts
+++ b/server/routes/auth.ts
@@ -6,7 +6,6 @@ import { UserType } from '@server/constants/user';
 import { getRepository } from '@server/datasource';
 import { User } from '@server/entity/User';
 import { startJobs } from '@server/job/schedule';
-import ImageProxy from '@server/lib/imageproxy';
 import { Permission } from '@server/lib/permissions';
 import { getSettings } from '@server/lib/settings';
 import logger from '@server/logger';
@@ -15,7 +14,6 @@ import { ApiError } from '@server/types/error';
 import { getHostname } from '@server/utils/getHostname';
 import * as EmailValidator from 'email-validator';
 import { Router } from 'express';
-import gravatarUrl from 'gravatar-url';
 import net from 'net';
 
 const authRoutes = Router();
@@ -328,12 +326,7 @@ authRoutes.post('/jellyfin', async (req, res, next) => {
             jellyfinDeviceId: deviceId,
             jellyfinAuthToken: account.AccessToken,
             permissions: Permission.ADMIN,
-            avatar: account.User.PrimaryImageTag
-              ? `/Users/${account.User.Id}/Images/Primary/?tag=${account.User.PrimaryImageTag}&quality=90`
-              : gravatarUrl(body.email || account.User.Name, {
-                  default: 'mm',
-                  size: 200,
-                }),
+            avatar: `/avatarproxy/${account.User.Id}`,
             userType: UserType.EMBY,
           });
 
@@ -347,12 +340,7 @@ authRoutes.post('/jellyfin', async (req, res, next) => {
             jellyfinDeviceId: deviceId,
             jellyfinAuthToken: account.AccessToken,
             permissions: Permission.ADMIN,
-            avatar: account.User.PrimaryImageTag
-              ? `/Users/${account.User.Id}/Images/Primary/?tag=${account.User.PrimaryImageTag}&quality=90`
-              : gravatarUrl(body.email || account.User.Name, {
-                  default: 'mm',
-                  size: 200,
-                }),
+            avatar: `/avatarproxy/${account.User.Id}`,
             userType: UserType.JELLYFIN,
           });
 
@@ -401,27 +389,7 @@ authRoutes.post('/jellyfin', async (req, res, next) => {
           jellyfinUsername: account.User.Name,
         }
       );
-      // Update the users avatar with their jellyfin profile pic (incase it changed)
-      if (account.User.PrimaryImageTag) {
-        const avatar = `/Users/${account.User.Id}/Images/Primary/?tag=${account.User.PrimaryImageTag}&quality=90`;
-        if (avatar !== user.avatar) {
-          const avatarProxy = new ImageProxy('avatar', '');
-          avatarProxy.clearCachedImage(user.avatar);
-        }
-        user.avatar = avatar;
-      } else {
-        const avatar = gravatarUrl(user.email || account.User.Name, {
-          default: 'mm',
-          size: 200,
-        });
-
-        if (avatar !== user.avatar) {
-          const avatarProxy = new ImageProxy('avatar', '');
-          avatarProxy.clearCachedImage(user.avatar);
-        }
-
-        user.avatar = avatar;
-      }
+      user.avatar = `/avatarproxy/${account.User.Id}`;
       user.jellyfinUsername = account.User.Name;
 
       if (user.username === account.User.Name) {
@@ -459,12 +427,7 @@ authRoutes.post('/jellyfin', async (req, res, next) => {
         jellyfinUserId: account.User.Id,
         jellyfinDeviceId: deviceId,
         permissions: settings.main.defaultPermissions,
-        avatar: account.User.PrimaryImageTag
-          ? `/Users/${account.User.Id}/Images/Primary/?tag=${account.User.PrimaryImageTag}&quality=90`
-          : gravatarUrl(body.email || account.User.Name, {
-              default: 'mm',
-              size: 200,
-            }),
+        avatar: `/avatarproxy/${account.User.Id}`,
         userType:
           settings.main.mediaServerType === MediaServerType.JELLYFIN
             ? UserType.JELLYFIN
diff --git a/server/routes/avatarproxy.ts b/server/routes/avatarproxy.ts
index e6f6f3b54..2d72e2f19 100644
--- a/server/routes/avatarproxy.ts
+++ b/server/routes/avatarproxy.ts
@@ -1,21 +1,39 @@
 import { MediaServerType } from '@server/constants/server';
+import { getRepository } from '@server/datasource';
+import { User } from '@server/entity/User';
 import ImageProxy from '@server/lib/imageproxy';
 import { getSettings } from '@server/lib/settings';
 import logger from '@server/logger';
+import { getAppVersion } from '@server/utils/appVersion';
 import { getHostname } from '@server/utils/getHostname';
 import { Router } from 'express';
+import gravatarUrl from 'gravatar-url';
 
 const router = Router();
 
-const avatarImageProxy = new ImageProxy('avatar', '');
-// Proxy avatar images
-router.get('/*', async (req, res) => {
-  let imagePath = '';
+let _avatarImageProxy: ImageProxy | null = null;
+async function initAvatarImageProxy() {
+  if (!_avatarImageProxy) {
+    const userRepository = getRepository(User);
+    const admin = await userRepository.findOne({
+      where: { id: 1 },
+      select: ['id', 'jellyfinUserId', 'jellyfinDeviceId'],
+      order: { id: 'ASC' },
+    });
+    const deviceId = admin?.jellyfinDeviceId;
+    const authToken = getSettings().jellyfin.apiKey;
+    _avatarImageProxy = new ImageProxy('avatar', '', {
+      headers: {
+        'X-Emby-Authorization': `MediaBrowser Client="Jellyseerr", Device="Jellyseerr", DeviceId="${deviceId}", Version="${getAppVersion()}", Token="${authToken}"`,
+      },
+    });
+  }
+  return _avatarImageProxy;
+}
+
+router.get('/:jellyfinUserId', async (req, res) => {
   try {
-    const jellyfinAvatar = req.url.match(
-      /(\/Users\/\w+\/Images\/Primary\/?\?tag=\w+&quality=90)$/
-    )?.[1];
-    if (!jellyfinAvatar) {
+    if (!req.params.jellyfinUserId.match(/^[a-f0-9]{32}$/)) {
       const mediaServerType = getSettings().main.mediaServerType;
       throw new Error(
         `Provided URL is not ${
@@ -26,10 +44,28 @@ router.get('/*', async (req, res) => {
       );
     }
 
-    const imageUrl = new URL(jellyfinAvatar, getHostname());
-    imagePath = imageUrl.toString();
+    const avatarImageCache = await initAvatarImageProxy();
 
-    const imageData = await avatarImageProxy.getImage(imagePath);
+    const user = await getRepository(User).findOne({
+      where: { jellyfinUserId: req.params.jellyfinUserId },
+    });
+
+    const fallbackUrl = gravatarUrl(user?.email || 'none', {
+      default: 'mm',
+      size: 200,
+    });
+    const jellyfinAvatarUrl = `${getHostname()}/UserImage?UserId=${
+      req.params.jellyfinUserId
+    }`;
+    let imageData = await avatarImageCache.getImage(
+      jellyfinAvatarUrl,
+      fallbackUrl
+    );
+
+    if (imageData.meta.extension === 'json') {
+      // this is a 404
+      imageData = await avatarImageCache.getImage(fallbackUrl);
+    }
 
     res.writeHead(200, {
       'Content-Type': `image/${imageData.meta.extension}`,
@@ -42,7 +78,6 @@ router.get('/*', async (req, res) => {
     res.end(imageData.imageBuffer);
   } catch (e) {
     logger.error('Failed to proxy avatar image', {
-      imagePath,
       errorMessage: e.message,
     });
   }
diff --git a/server/routes/settings/index.ts b/server/routes/settings/index.ts
index 3d6b6b0d3..c5a070d2d 100644
--- a/server/routes/settings/index.ts
+++ b/server/routes/settings/index.ts
@@ -32,7 +32,6 @@ import { getHostname } from '@server/utils/getHostname';
 import { Router } from 'express';
 import rateLimit from 'express-rate-limit';
 import fs from 'fs';
-import gravatarUrl from 'gravatar-url';
 import { escapeRegExp, merge, omit, set, sortBy } from 'lodash';
 import { rescheduleJob } from 'node-schedule';
 import path from 'path';
@@ -395,9 +394,7 @@ settingsRoutes.get('/jellyfin/users', async (req, res) => {
   const users = resp.users.map((user) => ({
     username: user.Name,
     id: user.Id,
-    thumb: user.PrimaryImageTag
-      ? `/Users/${user.Id}/Images/Primary/?tag=${user.PrimaryImageTag}&quality=90`
-      : gravatarUrl(user.Name, { default: 'mm', size: 200 }),
+    thumb: `/avatarproxy/${user.Id}`,
     email: user.Name,
   }));
 
diff --git a/server/routes/user/index.ts b/server/routes/user/index.ts
index 83ad0910b..2a29c0374 100644
--- a/server/routes/user/index.ts
+++ b/server/routes/user/index.ts
@@ -539,12 +539,7 @@ router.post(
             ).toString('base64'),
             email: jellyfinUser?.Name,
             permissions: settings.main.defaultPermissions,
-            avatar: jellyfinUser?.PrimaryImageTag
-              ? `/Users/${jellyfinUser.Id}/Images/Primary/?tag=${jellyfinUser.PrimaryImageTag}&quality=90`
-              : gravatarUrl(jellyfinUser?.Name ?? '', {
-                  default: 'mm',
-                  size: 200,
-                }),
+            avatar: `/avatarproxy/${jellyfinUser?.Id}`,
             userType:
               settings.main.mediaServerType === MediaServerType.JELLYFIN
                 ? UserType.JELLYFIN
diff --git a/src/components/Common/CachedImage/index.tsx b/src/components/Common/CachedImage/index.tsx
index b01a47bb4..a6d2fb001 100644
--- a/src/components/Common/CachedImage/index.tsx
+++ b/src/components/Common/CachedImage/index.tsx
@@ -25,11 +25,8 @@ const CachedImage = ({ src, type, ...props }: CachedImageProps) => {
         ? src.replace(/^https:\/\/image\.tmdb\.org\//, '/imageproxy/')
         : src;
   } else if (type === 'avatar') {
-    // jellyfin avatar (in any)
-    const jellyfinAvatar = src.match(
-      /(\/Users\/\w+\/Images\/Primary\/?\?tag=\w+&quality=90)$/
-    )?.[1];
-    imageUrl = jellyfinAvatar ? `/avatarproxy` + jellyfinAvatar : src;
+    // jellyfin avatar (if any)
+    imageUrl = src;
   } else {
     return null;
   }