rook-ceph

cluster/apps/data/bitwardenrs/volume.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn-backups
+  storageClassName: rook-ceph-block
   resources:
     requests:
       storage: 1Gi

cluster/apps/data/bookstack/volume.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn-backups
+  storageClassName: rook-ceph-block
   resources:
     requests:
       storage: 1Gi

cluster/apps/data/freshrss/volume.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn-backups
+  storageClassName: rook-ceph-block
   resources:
     requests:
       storage: 1Gi

cluster/apps/data/pgadmin/volume.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn-backups
+  storageClassName: rook-ceph-block
   resources:
     requests:
       storage: 1Gi

cluster/apps/data/recipes/helm-release.yaml

@@ -58,17 +58,22 @@ spec:
       tls:
         - hosts:
             - "recipes.${SECRET_CLUSTER_DOMAIN}"
-    persistence:
+
-      config:
+    additionalVolumes:
-        enabled: false
+      - name: files
-      media:
+        persistentVolumeClaim:
-        enabled: true
+          claimName: recipes-files
+      - name: recipes-config
+        configMap:
+          name: recipes-config
+    additionalVolumeMounts:
+      - name: files
         mountPath: /opt/recipes/mediafiles
-        existingClaim: recipes-media
+        subPath: media
-      static:
+      - name: files
-        enabled: true
         mountPath: /opt/recipes/staticfiles
-        existingClaim: recipes-static
+        subPath: static
+
     additionalContainers:
       - name: nginx
         image: nginx:1.19.10
@@ -76,10 +81,12 @@ spec:
           - containerPort: 80
             name: http
         volumeMounts:
-          - name: media
+          - name: files
             mountPath: "/media"
-          - name: static
+            subPath: media
+          - name: files
             mountPath: "/static"
+            subPath: static
           - name: recipes-config
             mountPath: /etc/nginx/nginx.conf
             subPath: nginx-config

cluster/apps/data/recipes/kustomization.yaml

@@ -2,4 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - helm-release.yaml
-  - volumes.yaml
+  - volume.yaml

cluster/apps/data/recipes/volume.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: recipes-files
+  namespace: data
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: rook-ceph-block
+  resources:
+    requests:
+      storage: 1Gi

cluster/apps/data/recipes/volumes.yaml

@@ -1,26 +0,0 @@
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: recipes-media
-  namespace: data
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: longhorn-backups
-  resources:
-    requests:
-      storage: 5Gi
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: recipes-static
-  namespace: data
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: longhorn-backups
-  resources:
-    requests:
-      storage: 1Gi

cluster/apps/data/resilio-sync/kustomization.yaml

@@ -2,4 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - statefulset.yaml
-  - volumes.yaml
+  - volume.yaml

cluster/apps/data/resilio-sync/statefulset.yaml

@@ -39,8 +39,9 @@ spec:
             - containerPort: 55555
               name: com-claude
           volumeMounts:
-            - name: config-claude
+            - name: config
               mountPath: /config
+              subPath: claude
             - name: sync-conf-claude
               mountPath: /config/sync.conf
               subPath: sync.conf
@@ -72,8 +73,9 @@ spec:
             - containerPort: 55556
               name: com-helene
           volumeMounts:
-            - name: config-helene
+            - name: config
               mountPath: /config
+              subPath: helene
             - name: sync-conf-helene
               mountPath: /config/sync.conf
               subPath: sync.conf
@@ -82,18 +84,15 @@ spec:
             - name: nfs-backups-data
               mountPath: /sync/backup
       volumes:
+        - name: config
+          persistentVolumeClaim:
+            claimName: resilio-sync-config
         - name: sync-conf-claude
           configMap:
             name: resilio-sync-claude-conf
         - name: sync-conf-helene
           configMap:
             name: resilio-sync-helene-conf
-        - name: config-claude
-          persistentVolumeClaim:
-            claimName: resilio-sync-config-claude
-        - name: config-helene
-          persistentVolumeClaim:
-            claimName: resilio-sync-config-helene
         - name: home-claude-data
           persistentVolumeClaim:
             claimName: nfs-home-claude-data

cluster/apps/data/resilio-sync/volume.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: resilio-sync-config
+  namespace: data
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: rook-ceph-block
+  resources:
+    requests:
+      storage: 1Gi

cluster/apps/data/resilio-sync/volumes.yaml

@@ -1,26 +0,0 @@
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: resilio-sync-config-claude
-  namespace: data
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: longhorn-backups
-  resources:
-    requests:
-      storage: 1Gi
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: resilio-sync-config-helene
-  namespace: data
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: longhorn-backups
-  resources:
-    requests:
-      storage: 1Gi

cluster/apps/data/vikunja/volume.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn-backups
+  storageClassName: rook-ceph-block
   resources:
     requests:
       storage: 1Gi

cluster/apps/development/gitea/volume.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn-backups
+  storageClassName: rook-ceph-block
   resources:
     requests:
-      storage: 15Gi
+      storage: 10Gi

cluster/apps/home/esphome/volume.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn-backups
+  storageClassName: rook-ceph-block
   resources:
     requests:
       storage: 10Gi

cluster/apps/home/home-assistant/volume.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn-backups
+  storageClassName: rook-ceph-block
   resources:
     requests:
       storage: 10Gi

cluster/apps/home/node-red/volume.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn-backups
+  storageClassName: rook-ceph-block
   resources:
     requests:
       storage: 10Gi

cluster/apps/home/zigbee2mqtt/helm-release.yaml

@@ -27,7 +27,7 @@ spec:
       homeassistant: true
       device_options:
         retain: true
-      permit_join: false
+      permit_join: true
       mqtt:
         base_topic: zigbee2mqtt
         server: "mqtt://vernemq"

cluster/apps/home/zigbee2mqtt/volume.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn-backups
+  storageClassName: rook-ceph-block
   resources:
     requests:
-      storage: 10Gi
+      storage: 1Gi

cluster/apps/home/zwavejs2mqtt/volume.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn-backups
+  storageClassName: rook-ceph-block
   resources:
     requests:
       storage: 10Gi

cluster/apps/kasten-io/k10/helm-release.yaml

@@ -24,7 +24,7 @@ spec:
       email: "${SECRET_CLUSTER_DOMAIN_EMAIL}"
     global:
       persistence:
-        storageClass: longhorn
+        storageClass: rook-ceph-block
     auth:
       tokenAuth:
         enabled: true

cluster/apps/media/bazarr/volume.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn-backups
+  storageClassName: rook-ceph-block
   resources:
     requests:
-      storage: 20Gi
+      storage: 1Gi

cluster/apps/media/flood/volume.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn-backups
+  storageClassName: rook-ceph-block
   resources:
     requests:
       storage: 1Gi

cluster/apps/media/jackett/volume.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn-backups
+  storageClassName: rook-ceph-block
   resources:
     requests:
       storage: 1Gi

cluster/apps/media/jellyfin/volume.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn-backups
+  storageClassName: rook-ceph-block
   resources:
     requests:
-      storage: 20Gi
+      storage: 30Gi

cluster/apps/media/lidarr/volume.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn-backups
+  storageClassName: rook-ceph-block
   resources:
     requests:
       storage: 20Gi

cluster/apps/media/lychee/helm-release.yaml

@@ -21,14 +21,17 @@ spec:
       repository: lycheeorg/lychee-laravel
       pullPolicy: IfNotPresent
       tag: v4.2.2
+
     strategy:
       type: Recreate
+
     service:
       port:
         port: 80
       annotations:
         prometheus.io/probe: "true"
         prometheus.io/protocol: http
+
     ingress:
       enabled: true
       annotations:
@@ -44,6 +47,7 @@ spec:
       tls:
         - hosts:
             - "lychee.${SECRET_CLUSTER_DOMAIN}"
+
     env:
       PHP_TZ: Europe/Paris
       DB_CONNECTION: pgsql
@@ -52,20 +56,22 @@ spec:
       DB_DATABASE: lychee
       DB_USERNAME: lychee
       DB_PASSWORD: ${SECRET_LYCHEE_DB_PASSWORD}
+
     persistence:
-      config:
-        enabled: true
-        mountPath: /conf
-        existingClaim: lychee-config
-      uploads:
-        enabled: true
-        mountPath: /uploads
-        existingClaim: lychee-uploads
-      sym:
-        enabled: true
-        mountPath: /sym
-        existingClaim: lychee-sym
       photo:
         enabled: true
         mountPath: /mnt/storage/photo
         existingClaim: nfs-photo-media
+        readOnly: true
+
+    additionalVolumes:
+      - name: files
+        persistentVolumeClaim:
+          claimName: lychee-files
+    additionalVolumeMounts:
+      - name: files
+        mountPath: /uploads
+        subPath: uploads
+      - name: files
+        mountPath: /sym
+        subPath: sym

cluster/apps/media/lychee/kustomization.yaml

@@ -2,4 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - helm-release.yaml
-  - volumes.yaml
+  - volume.yaml

cluster/apps/media/lychee/volume.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: lychee-files
+  namespace: media
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: rook-ceph-block
+  resources:
+    requests:
+      storage: 100Gi

cluster/apps/media/lychee/volumes.yaml

@@ -1,39 +0,0 @@
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: lychee-config
-  namespace: media
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: longhorn-backups
-  resources:
-    requests:
-      storage: 1Gi
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: lychee-uploads
-  namespace: media
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: longhorn-backups
-  resources:
-    requests:
-      storage: 50Gi
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: lychee-sym
-  namespace: media
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: longhorn-backups
-  resources:
-    requests:
-      storage: 1Gi

cluster/apps/media/navidrome/volume.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn-backups
+  storageClassName: rook-ceph-block
   resources:
     requests:
       storage: 1Gi

cluster/apps/media/prowlarr/volume.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn-backups
+  storageClassName: rook-ceph-block
   resources:
     requests:
       storage: 1Gi

cluster/apps/media/pyload/volume.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn-backups
+  storageClassName: rook-ceph-block
   resources:
     requests:
       storage: 1Gi

cluster/apps/media/qbittorrent/volumes.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn-backups
+  storageClassName: rook-ceph-block
   resources:
     requests:
       storage: 1Gi

cluster/apps/media/radarr/helm-release.yaml

@@ -66,6 +66,11 @@ spec:
         forecastle.stakater.com/appName: "Radarr"
         forecastle.stakater.com/icon: "https://raw.githubusercontent.com/Radarr/Radarr/develop/Logo/256.png"
         forecastle.stakater.com/network-restricted: "true"
+        # -- Nginx client Body Buffer Size
+        nginx.ingress.kubernetes.io/client-body-buffer-size: "20m"
+        # -- Nginx Proxy Body Size
+        nginx.ingress.kubernetes.io/proxy-body-size: "20m"
+        nginx.ingress.kubernetes.io/proxy-buffering: "off"
       hosts:
         - host: radarr.${SECRET_CLUSTER_DOMAIN}
           paths:

cluster/apps/media/radarr/volume.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn-backups
+  storageClassName: rook-ceph-block
   resources:
     requests:
       storage: 20Gi

cluster/apps/media/sonarr/helm-release.yaml

@@ -46,6 +46,11 @@ spec:
         forecastle.stakater.com/appName: "Sonarr"
         forecastle.stakater.com/icon: "https://raw.githubusercontent.com/Sonarr/Sonarr/develop/Logo/256.png"
         forecastle.stakater.com/network-restricted: "true"
+        # -- Nginx client Body Buffer Size
+        nginx.ingress.kubernetes.io/client-body-buffer-size: "20m"
+        # -- Nginx Proxy Body Size
+        nginx.ingress.kubernetes.io/proxy-body-size: "20m"
+        nginx.ingress.kubernetes.io/proxy-buffering: "off"
       hosts:
         - host: sonarr.${SECRET_CLUSTER_DOMAIN}
           paths:

cluster/apps/media/sonarr/volume.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn-backups
+  storageClassName: rook-ceph-block
   resources:
     requests:
       storage: 20Gi

cluster/apps/media/tdarr/volume.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn-backups
+  storageClassName: rook-ceph-block
   resources:
     requests:
-      storage: 2Gi
+      storage: 5Gi

cluster/apps/monitoring/kube-prometheus-stack/helm-release.yaml

@@ -65,7 +65,7 @@ spec:
         storage:
           volumeClaimTemplate:
             spec:
-              storageClassName: longhorn
+              storageClassName: rook-ceph-block
               resources:
                 requests:
                   storage: 10Gi
@@ -92,9 +92,6 @@ spec:
           kubernetes-custom:
             url: https://raw.githubusercontent.com/auricom/home-cluster/main/cluster/apps/monitoring/kube-prometheus-stack/grafana-dashboards/kubernetes-custom.json
             datasource: Prometheus
-          longhorn:
-            url: https://raw.githubusercontent.com/auricom/home-cluster/main/cluster/apps/monitoring/kube-prometheus-stack/grafana-dashboards/longhorn.json
-            datasource: Prometheus
       deploymentStrategy:
         type: Recreate
       persistence:
@@ -198,7 +195,7 @@ spec:
         storageSpec:
           volumeClaimTemplate:
             spec:
-              storageClassName: longhorn
+              storageClassName: rook-ceph-block
               resources:
                 requests:
                   storage: 10Gi

cluster/apps/networking/unifi/volume.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
-  storageClassName: longhorn-backups
+  storageClassName: rook-ceph-block
   resources:
     requests:
       storage: 10Gi

cluster/base/flux-system/charts/kustomization.yaml

@@ -21,6 +21,7 @@ resources:
   - nicholaswilde.yaml
   - node-feature-discovery.yaml
   - prometheus-community-charts.yaml
+  - rook-ceph-charts.yaml
   - runix-charts.yaml
   - stakater-charts.yaml
   - twuni-charts.yaml

cluster/base/flux-system/charts/rook-ceph-charts.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: HelmRepository
+metadata:
+  name: rook-ceph-charts
+  namespace: flux-system
+spec:
+  interval: 10m
+  url: https://charts.rook.io/release
+  timeout: 3m

cluster/core/infrastructure/kustomization.yaml

@@ -7,4 +7,5 @@ resources:
   - intel-gpu-plugin
   - longhorn-system
   - node-feature-discovery
+  - rook-ceph
   - system-upgrade

cluster/core/infrastructure/rook-ceph/dashboard/ingress.yaml

@@ -0,0 +1,29 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: rook-ceph-mgr-dashboard
+  namespace: rook-ceph
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    nginx.ingress.kubernetes.io/backend-protocol: "HTTP"
+    nginx.ingress.kubernetes.io/auth-url: "http://authelia.networking.svc.cluster.local/api/verify"
+    nginx.ingress.kubernetes.io/auth-signin: "https://login.${SECRET_CLUSTER_DOMAIN}/"
+  labels:
+    app.kubernetes.io/instance: rook-ceph-mgr-dashboard
+    app.kubernetes.io/name: rook-ceph-mgr-dashboard
+spec:
+  rules:
+    - host: "rook.${SECRET_CLUSTER_DOMAIN}"
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: rook-ceph-mgr-dashboard
+                port:
+                  name: http-dashboard
+  tls:
+    - hosts:
+        - "rook.${SECRET_CLUSTER_DOMAIN}"

cluster/core/infrastructure/rook-ceph/dashboard/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ingress.yaml

cluster/core/infrastructure/rook-ceph/helm-release.yaml

@@ -0,0 +1,35 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: rook-ceph
+  namespace: rook-ceph
+spec:
+  interval: 5m
+  chart:
+    spec:
+      # renovate: registryUrl=https://charts.rook.io/release
+      chart: rook-ceph
+      version: v1.5.10
+      sourceRef:
+        kind: HelmRepository
+        name: rook-ceph-charts
+        namespace: flux-system
+  values:
+    crds:
+      enabled: false
+    csi:
+      kubeletDirPath: /var/lib/kubelet
+      pluginTolerations:
+        - key: "node-role.kubernetes.io/master"
+          operator: "Exists"
+        - effect: NoExecute
+          operator: Exists
+        - effect: NoSchedule
+          operator: Exists
+    resources:
+      requests:
+        cpu: 100m
+        memory: 128Mi
+      limits:
+        cpu: 1000m

cluster/core/infrastructure/rook-ceph/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - rbac.yaml
+  - helm-release.yaml
+  - storage
+  - rook-direct-mount
+  - servicemonitor
+  - snapshot-controller
+  - dashboard

cluster/core/infrastructure/rook-ceph/rbac.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: "rook-ceph-system-psp-user"
+  labels:
+    operator: rook
+    storage-backend: ceph
+rules:
+  - apiGroups:
+      - policy
+    resources:
+      - podsecuritypolicies
+    resourceNames:
+      - 00-rook-ceph-operator
+    verbs:
+      - use

cluster/core/infrastructure/rook-ceph/rook-direct-mount/deployment.yaml

@@ -0,0 +1,65 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: rook-direct-mount
+  namespace: rook-ceph
+  labels:
+    app: rook-direct-mount
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: rook-direct-mount
+  template:
+    metadata:
+      labels:
+        app: rook-direct-mount
+    spec:
+      dnsPolicy: ClusterFirstWithHostNet
+      containers:
+        - name: rook-direct-mount
+          image: rook/ceph:v1.6.0
+          command: ["/tini"]
+          args: ["-g", "--", "/usr/local/bin/toolbox.sh"]
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: ROOK_CEPH_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: rook-ceph-mon
+                  key: ceph-username
+            - name: ROOK_CEPH_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: rook-ceph-mon
+                  key: ceph-secret
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - mountPath: /dev
+              name: dev
+            - mountPath: /sys/bus
+              name: sysbus
+            - mountPath: /lib/modules
+              name: libmodules
+            - name: mon-endpoint-volume
+              mountPath: /etc/rook
+      # if hostNetwork: false, the "rbd map" command hangs, see https://github.com/rook/rook/issues/2021
+      hostNetwork: true
+      volumes:
+        - name: dev
+          hostPath:
+            path: /dev
+        - name: sysbus
+          hostPath:
+            path: /sys/bus
+        - name: libmodules
+          hostPath:
+            path: /lib/modules
+        - name: mon-endpoint-volume
+          configMap:
+            name: rook-ceph-mon-endpoints
+            items:
+              - key: data
+                path: mon-endpoints

cluster/core/infrastructure/rook-ceph/rook-direct-mount/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - deployment.yaml

cluster/core/infrastructure/rook-ceph/servicemonitor/csi-metrics.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: csi-metrics
+  namespace: rook-ceph
+  labels:
+    team: rook
+spec:
+  namespaceSelector:
+    matchNames:
+      - rook-ceph
+  selector:
+    matchLabels:
+      app: csi-metrics
+  endpoints:
+    - port: csi-http-metrics
+      path: /metrics
+      interval: 5s

cluster/core/infrastructure/rook-ceph/servicemonitor/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - csi-metrics.yaml
+  - rook-ceph-mgr.yaml

cluster/core/infrastructure/rook-ceph/servicemonitor/rook-ceph-mgr.yaml

@@ -0,0 +1,20 @@
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: rook-ceph-mgr
+  namespace: rook-ceph
+  labels:
+    team: rook
+spec:
+  namespaceSelector:
+    matchNames:
+      - rook-ceph
+  selector:
+    matchLabels:
+      app: rook-ceph-mgr
+      rook_cluster: rook-ceph
+  endpoints:
+    - port: http-metrics
+      path: /metrics
+      interval: 5s

cluster/core/infrastructure/rook-ceph/snapshot-controller/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - rbac.yaml
+  - statefulset.yaml

cluster/core/infrastructure/rook-ceph/snapshot-controller/rbac.yaml

@@ -0,0 +1,73 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: snapshot-controller
+  namespace: rook-ceph
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: snapshot-controller-runner
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots/status"]
+    verbs: ["update"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: snapshot-controller-role
+subjects:
+  - kind: ServiceAccount
+    name: snapshot-controller
+    namespace: rook-ceph
+roleRef:
+  kind: ClusterRole
+  name: snapshot-controller-runner
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: rook-ceph
+  name: snapshot-controller-leaderelection
+rules:
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: snapshot-controller-leaderelection
+  namespace: rook-ceph
+subjects:
+  - kind: ServiceAccount
+    name: snapshot-controller
+    namespace: rook-ceph
+roleRef:
+  kind: Role
+  name: snapshot-controller-leaderelection
+  apiGroup: rbac.authorization.k8s.io

cluster/core/infrastructure/rook-ceph/snapshot-controller/statefulset.yaml

@@ -0,0 +1,25 @@
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: snapshot-controller
+  namespace: rook-ceph
+spec:
+  serviceName: "snapshot-controller"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: snapshot-controller
+  template:
+    metadata:
+      labels:
+        app: snapshot-controller
+    spec:
+      serviceAccount: snapshot-controller
+      containers:
+        - name: snapshot-controller
+          image: k8s.gcr.io/sig-storage/snapshot-controller:v4.0.0
+          args:
+            - "--v=5"
+            - "--leader-election=false"
+          imagePullPolicy: IfNotPresent

cluster/core/infrastructure/rook-ceph/storage/cephblockpool.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: replicapool
+  namespace: rook-ceph
+spec:
+  failureDomain: host
+  replicated:
+    size: 3
+    requireSafeReplicaSize: true

cluster/core/infrastructure/rook-ceph/storage/cephcluster.yaml

@@ -0,0 +1,71 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: rook-ceph
+  namespace: rook-ceph
+spec:
+  cephVersion:
+    image: ceph/ceph:v15.2.10
+    allowUnsupported: false
+  dataDirHostPath: /var/lib/rook
+  skipUpgradeChecks: false
+  continueUpgradeAfterChecksEvenIfNotHealthy: false
+  removeOSDsIfOutAndSafeToRemove: false
+  mon:
+    count: 3
+    allowMultiplePerNode: false
+  monitoring:
+    enabled: true
+    rulesNamespace: rook-ceph
+  network:
+  crashCollector:
+    disable: false
+  cleanupPolicy:
+    confirmation: ""
+    sanitizeDisks:
+      method: quick
+      dataSource: zero
+      iteration: 1
+  mgr:
+    modules:
+      - name: pg_autoscaler
+        enabled: true
+  dashboard:
+    enabled: true
+    port: 7000
+    ssl: false
+  disruptionManagement:
+    managePodBudgets: false
+    osdMaintenanceTimeout: 30
+    manageMachineDisruptionBudgets: false
+    machineDisruptionBudgetNamespace: openshift-machine-api
+  resources:
+    mon:
+      requests:
+        cpu: 35m
+        memory: 800Mi
+      limits:
+        memory: 1024Mi
+    osd:
+      requests:
+        cpu: 35m
+        memory: 2048Mi
+      limits:
+        memory: 4096Mi
+  storage:
+    useAllNodes: false
+    useAllDevices: false
+    config:
+      metadataDevice:
+      osdsPerDevice: "1"
+    nodes:
+      - name: "k3s-worker1"
+        devices:
+          - name: "nvme0n1"
+      - name: "k3s-worker2"
+        devices:
+          - name: "nvme0n1"
+      - name: "k3s-worker3"
+        devices:
+          - name: "nvme0n1"

cluster/core/infrastructure/rook-ceph/storage/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - cephblockpool.yaml
+  - cephcluster.yaml
+  - storageclass.yaml
+  - volumesnapshotclass.yaml

cluster/core/infrastructure/rook-ceph/storage/storageclass.yaml

@@ -0,0 +1,22 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: rook-ceph-block
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+provisioner: rook-ceph.rbd.csi.ceph.com
+parameters:
+  clusterID: rook-ceph
+  pool: replicapool
+  imageFormat: "2"
+  imageFeatures: layering
+  csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
+  csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
+  csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
+  csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
+  csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
+  csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
+  csi.storage.k8s.io/fstype: ext4
+reclaimPolicy: Delete
+allowVolumeExpansion: true

cluster/core/infrastructure/rook-ceph/storage/volumesnapshotclass.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: snapshot.storage.k8s.io/v1beta1
+kind: VolumeSnapshotClass
+metadata:
+  name: csi-rbdplugin-snapclass
+  annotations:
+    k10.kasten.io/is-snapshot-class: "true"
+driver: rook-ceph.rbd.csi.ceph.com
+parameters:
+  # Specify a string that identifies your cluster. Ceph CSI supports any
+  # unique string. When Ceph CSI is deployed by Rook use the Rook namespace,
+  # for example "rook-ceph".
+  clusterID: rook-ceph
+  csi.storage.k8s.io/snapshotter-secret-name: rook-csi-rbd-provisioner
+  csi.storage.k8s.io/snapshotter-secret-namespace: rook-ceph
+deletionPolicy: Delete

cluster/core/namespaces/rook-ceph.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: rook-ceph