fix: rook-direct-mount

2025-09-30 15:37:44 +02:00 · 2021-12-12 20:19:16 +01:00
parent 291d02f76d
commit 877ac1203d
19 changed files with 92 additions and 385 deletions
--- a/cluster/base-custom/crds/rook-ceph/crds.yaml
+++ b/cluster/base-custom/crds/rook-ceph/crds.yaml
@@ -5,7 +5,7 @@ metadata:
  name: rook-ceph-source
  namespace: flux-system
 spec:
-  interval: 1h
+  interval: 30m
  url: https://github.com/rook/rook.git
  ref:
    # renovate: registryUrl=https://charts.rook.io/release chart=rook-ceph
@@ -14,7 +14,7 @@ spec:
    # exclude all
    /*
    # path to crds
-    !/cluster/examples/kubernetes/ceph/crds.yaml
+    !/deploy/examples/crds.yaml
 ---
 apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
 kind: Kustomization
@@ -24,43 +24,7 @@ metadata:
 spec:
  interval: 15m
  prune: false
+  wait: true
  sourceRef:
    kind: GitRepository
    name: rook-ceph-source
-  healthChecks:
-    - apiVersion: apiextensions.k8s.io/v1
-      kind: CustomResourceDefinition
-      name: cephblockpools.ceph.rook.io
-    - apiVersion: apiextensions.k8s.io/v1
-      kind: CustomResourceDefinition
-      name: cephclients.ceph.rook.io
-    - apiVersion: apiextensions.k8s.io/v1
-      kind: CustomResourceDefinition
-      name: cephclusters.ceph.rook.io
-    - apiVersion: apiextensions.k8s.io/v1
-      kind: CustomResourceDefinition
-      name: cephfilesystems.ceph.rook.io
-    - apiVersion: apiextensions.k8s.io/v1
-      kind: CustomResourceDefinition
-      name: cephnfses.ceph.rook.io
-    - apiVersion: apiextensions.k8s.io/v1
-      kind: CustomResourceDefinition
-      name: cephobjectrealms.ceph.rook.io
-    - apiVersion: apiextensions.k8s.io/v1
-      kind: CustomResourceDefinition
-      name: cephobjectstores.ceph.rook.io
-    - apiVersion: apiextensions.k8s.io/v1
-      kind: CustomResourceDefinition
-      name: cephobjectstoreusers.ceph.rook.io
-    - apiVersion: apiextensions.k8s.io/v1
-      kind: CustomResourceDefinition
-      name: cephobjectzonegroups.ceph.rook.io
-    - apiVersion: apiextensions.k8s.io/v1
-      kind: CustomResourceDefinition
-      name: cephobjectzones.ceph.rook.io
-    - apiVersion: apiextensions.k8s.io/v1
-      kind: CustomResourceDefinition
-      name: cephrbdmirrors.ceph.rook.io
-    - apiVersion: apiextensions.k8s.io/v1
-      kind: CustomResourceDefinition
-      name: volumes.rook.io
--- a/cluster/core/rook-ceph/cluster/helm-release.yaml
+++ b/cluster/core/rook-ceph/cluster/helm-release.yaml
@@ -0,0 +1,78 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: rook-ceph-cluster
+  namespace: rook-ceph
+spec:
+  interval: 5m
+  chart:
+    spec:
+      # renovate: registryUrl=https://charts.rook.io/release
+      chart: rook-ceph-cluster
+      version: v1.8.0
+      sourceRef:
+        kind: HelmRepository
+        name: rook-ceph-charts
+        namespace: flux-system
+  dependsOn:
+    - name: rook-ceph
+  values:
+    monitoring:
+      enabled: true
+    ingress:
+      dashboard:
+        annotations:
+          kubernetes.io/ingress.class: "nginx"
+        host:
+          name: "rook.${SECRET_CLUSTER_DOMAIN}"
+          path: "/"
+        tls:
+          - hosts:
+              - "rook.${SECRET_CLUSTER_DOMAIN}"
+    cephClusterSpec:
+      mgr:
+        count: 1
+      dashboard:
+        enabled: true
+        urlPrefix: /
+        ssl: false
+      storage:
+        useAllNodes: false
+        useAllDevices: false
+        config:
+          osdsPerDevice: "1"
+        nodes:
+          - name: "k3s-worker1"
+            devices:
+              - name: "nvme0n1"
+          - name: "k3s-worker2"
+            devices:
+              - name: "nvme0n1"
+          - name: "k3s-worker3"
+            devices:
+              - name: "nvme0n1"
+    cephBlockPools:
+      - name: ceph-blockpool
+        spec:
+          failureDomain: host
+          replicated:
+            size: 3
+        storageClass:
+          enabled: true
+          name: rook-ceph-block
+          isDefault: true
+          reclaimPolicy: Delete
+          allowVolumeExpansion: true
+          parameters:
+            imageFormat: "2"
+            imageFeatures: layering
+            csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
+            csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
+            csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
+            csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
+            csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
+            csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
+            csi.storage.k8s.io/fstype: ext4
+    cephFileSystems: []
+    cephObjectStores: []
--- a/cluster/core/rook-ceph/dashboard/kustomization.yaml
+++ b/cluster/core/rook-ceph/dashboard/kustomization.yaml
@@ -1,4 +1,4 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - ingress.yaml
+  - helm-release.yaml
--- a/cluster/core/rook-ceph/dashboard/ingress.yaml
+++ b/cluster/core/rook-ceph/dashboard/ingress.yaml
@@ -1,28 +0,0 @@
---
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: rook-ceph-mgr-dashboard
-  namespace: rook-ceph
-  # annotations:
-    # traefik.ingress.kubernetes.io/router.entrypoints: "websecure"
-  labels:
-    app.kubernetes.io/instance: rook-ceph-mgr-dashboard
-    app.kubernetes.io/name: rook-ceph-mgr-dashboard
-spec:
-  ingressClassName: "nginx"
-  rules:
-    - host: "rook.${SECRET_CLUSTER_DOMAIN}"
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: rook-ceph-mgr-dashboard
-                port:
-                  name: http-dashboard
-  tls:
-    - hosts:
-        - "rook.${SECRET_CLUSTER_DOMAIN}"
-      secretName: "${SECRET_CLUSTER_CERTIFICATE_DEFAULT}"
--- a/cluster/core/rook-ceph/kustomization.yaml
+++ b/cluster/core/rook-ceph/kustomization.yaml
@@ -2,10 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - namespace.yaml
-  - rbac.yaml
-  - helm-release.yaml
-  - storage
+  - operator
+  - cluster
  - rook-direct-mount
-  - servicemonitor
-  - snapshot-controller
-  - dashboard
--- a/cluster/core/rook-ceph/operator/helm-release.yaml
+++ b/cluster/core/rook-ceph/operator/helm-release.yaml
@@ -19,14 +19,12 @@ spec:
    crds:
      enabled: false
    csi:
-      kubeletDirPath: /var/lib/kubelet
-      pluginTolerations:
-        - key: "node-role.kubernetes.io/control-plane"
-          operator: "Exists"
-          effect: "NoSchedule"
+      enableCephfsDriver: false
+      enableCephfsSnapshotter: false
    resources:
      requests:
        cpu: 100m
        memory: 128Mi
      limits:
        cpu: 1000m
+        memory: 256Mi
--- a/cluster/core/rook-ceph/snapshot-controller/kustomization.yaml
+++ b/cluster/core/rook-ceph/snapshot-controller/kustomization.yaml
@@ -1,5 +1,4 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - rbac.yaml
-  - statefulset.yaml
+  - helm-release.yaml
--- a/cluster/core/rook-ceph/rbac.yaml
+++ b/cluster/core/rook-ceph/rbac.yaml
@@ -1,17 +0,0 @@
---
-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: ClusterRole
-metadata:
-  name: "rook-ceph-system-psp-user"
-  labels:
-    operator: rook
-    storage-backend: ceph
-rules:
-  - apiGroups:
-      - policy
-    resources:
-      - podsecuritypolicies
-    resourceNames:
-      - 00-rook-ceph-operator
-    verbs:
-      - use
--- a/cluster/core/rook-ceph/rook-direct-mount/deployment.yaml
+++ b/cluster/core/rook-ceph/rook-direct-mount/deployment.yaml
@@ -20,8 +20,7 @@ spec:
      containers:
        - name: rook-direct-mount
          image: rook/ceph:v1.8.0
-          command: ["/tini"]
-          args: ["-g", "--", "/usr/local/bin/toolbox.sh"]
+          command: ["/usr/local/bin/toolbox.sh"]
          imagePullPolicy: IfNotPresent
          env:
            - name: ROOK_CEPH_USERNAME
@@ -45,6 +44,9 @@ spec:
              name: libmodules
            - name: mon-endpoint-volume
              mountPath: /etc/rook
+      securityContext:
+        runAsUser: 0
+        runAsGroup: 0
      # if hostNetwork: false, the "rbd map" command hangs, see https://github.com/rook/rook/issues/2021
      hostNetwork: true
      volumes:
--- a/cluster/core/rook-ceph/servicemonitor/csi-metrics.yaml
+++ b/cluster/core/rook-ceph/servicemonitor/csi-metrics.yaml
@@ -1,19 +0,0 @@
---
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: csi-metrics
-  namespace: rook-ceph
-  labels:
-    team: rook
-spec:
-  namespaceSelector:
-    matchNames:
-      - rook-ceph
-  selector:
-    matchLabels:
-      app: csi-metrics
-  endpoints:
-    - port: csi-http-metrics
-      path: /metrics
-      interval: 5s
--- a/cluster/core/rook-ceph/servicemonitor/kustomization.yaml
+++ b/cluster/core/rook-ceph/servicemonitor/kustomization.yaml
@@ -1,5 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - csi-metrics.yaml
-  - rook-ceph-mgr.yaml
--- a/cluster/core/rook-ceph/servicemonitor/rook-ceph-mgr.yaml
+++ b/cluster/core/rook-ceph/servicemonitor/rook-ceph-mgr.yaml
@@ -1,20 +0,0 @@
---
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: rook-ceph-mgr
-  namespace: rook-ceph
-  labels:
-    team: rook
-spec:
-  namespaceSelector:
-    matchNames:
-      - rook-ceph
-  selector:
-    matchLabels:
-      app: rook-ceph-mgr
-      rook_cluster: rook-ceph
-  endpoints:
-    - port: http-metrics
-      path: /metrics
-      interval: 5s
--- a/cluster/core/rook-ceph/snapshot-controller/rbac.yaml
+++ b/cluster/core/rook-ceph/snapshot-controller/rbac.yaml
@@ -1,73 +0,0 @@
---
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: snapshot-controller
-  namespace: rook-ceph
---
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: snapshot-controller-runner
-rules:
-  - apiGroups: [""]
-    resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["persistentvolumeclaims"]
-    verbs: ["get", "list", "watch", "update"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["storageclasses"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["events"]
-    verbs: ["list", "watch", "create", "update", "patch"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshotclasses"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshotcontents"]
-    verbs: ["create", "get", "list", "watch", "update", "delete"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshots"]
-    verbs: ["get", "list", "watch", "update"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshots/status"]
-    verbs: ["update"]
---
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: snapshot-controller-role
-subjects:
-  - kind: ServiceAccount
-    name: snapshot-controller
-    namespace: rook-ceph
-roleRef:
-  kind: ClusterRole
-  name: snapshot-controller-runner
-  apiGroup: rbac.authorization.k8s.io
---
-kind: Role
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  namespace: rook-ceph
-  name: snapshot-controller-leaderelection
-rules:
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "watch", "list", "delete", "update", "create"]
---
-kind: RoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: snapshot-controller-leaderelection
-  namespace: rook-ceph
-subjects:
-  - kind: ServiceAccount
-    name: snapshot-controller
-    namespace: rook-ceph
-roleRef:
-  kind: Role
-  name: snapshot-controller-leaderelection
-  apiGroup: rbac.authorization.k8s.io
--- a/cluster/core/rook-ceph/snapshot-controller/statefulset.yaml
+++ b/cluster/core/rook-ceph/snapshot-controller/statefulset.yaml
@@ -1,25 +0,0 @@
---
-kind: StatefulSet
-apiVersion: apps/v1
-metadata:
-  name: snapshot-controller
-  namespace: rook-ceph
-spec:
-  serviceName: "snapshot-controller"
-  replicas: 1
-  selector:
-    matchLabels:
-      app: snapshot-controller
-  template:
-    metadata:
-      labels:
-        app: snapshot-controller
-    spec:
-      serviceAccount: snapshot-controller
-      containers:
-        - name: snapshot-controller
-          image: k8s.gcr.io/sig-storage/snapshot-controller:v4.2.1
-          args:
-            - "--v=5"
-            - "--leader-election=false"
-          imagePullPolicy: IfNotPresent
--- a/cluster/core/rook-ceph/storage/cephblockpool.yaml
+++ b/cluster/core/rook-ceph/storage/cephblockpool.yaml
@@ -1,11 +0,0 @@
---
-apiVersion: ceph.rook.io/v1
-kind: CephBlockPool
-metadata:
-  name: replicapool
-  namespace: rook-ceph
-spec:
-  failureDomain: host
-  replicated:
-    size: 3
-    requireSafeReplicaSize: true
--- a/cluster/core/rook-ceph/storage/cephcluster.yaml
+++ b/cluster/core/rook-ceph/storage/cephcluster.yaml
@@ -1,70 +0,0 @@
---
-apiVersion: ceph.rook.io/v1
-kind: CephCluster
-metadata:
-  name: rook-ceph
-  namespace: rook-ceph
-spec:
-  cephVersion:
-    image: quay.io/ceph/ceph:v16.2.7
-    allowUnsupported: false
-  dataDirHostPath: /var/lib/rook
-  skipUpgradeChecks: false
-  continueUpgradeAfterChecksEvenIfNotHealthy: false
-  removeOSDsIfOutAndSafeToRemove: false
-  mon:
-    count: 3
-    allowMultiplePerNode: false
-  monitoring:
-    enabled: true
-    rulesNamespace: rook-ceph
-  network:
-  crashCollector:
-    disable: false
-  cleanupPolicy:
-    confirmation: ""
-    sanitizeDisks:
-      method: quick
-      dataSource: zero
-      iteration: 1
-  mgr:
-    modules:
-      - name: pg_autoscaler
-        enabled: true
-  dashboard:
-    enabled: true
-    port: 7000
-    ssl: false
-  disruptionManagement:
-    managePodBudgets: false
-    osdMaintenanceTimeout: 30
-    manageMachineDisruptionBudgets: false
-    machineDisruptionBudgetNamespace: openshift-machine-api
-  resources:
-    mon:
-      requests:
-        cpu: 35m
-        memory: 800Mi
-      limits:
-        memory: 1024Mi
-    osd:
-      requests:
-        cpu: 35m
-        memory: 2048Mi
-      limits:
-        memory: 4096Mi
-  storage:
-    useAllNodes: false
-    useAllDevices: false
-    config:
-      osdsPerDevice: "1"
-    nodes:
-      - name: "k3s-worker1"
-        devices:
-          - name: "nvme0n1"
-      - name: "k3s-worker2"
-        devices:
-          - name: "nvme0n1"
-      - name: "k3s-worker3"
-        devices:
-          - name: "nvme0n1"
--- a/cluster/core/rook-ceph/storage/kustomization.yaml
+++ b/cluster/core/rook-ceph/storage/kustomization.yaml
@@ -1,7 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - cephblockpool.yaml
-  - cephcluster.yaml
-  - storageclass.yaml
-  - volumesnapshotclass.yaml
--- a/cluster/core/rook-ceph/storage/storageclass.yaml
+++ b/cluster/core/rook-ceph/storage/storageclass.yaml
@@ -1,39 +0,0 @@
---
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: rook-ceph-block
-  annotations:
-    storageclass.kubernetes.io/is-default-class: "true"
-provisioner: rook-ceph.rbd.csi.ceph.com
-parameters:
-  clusterID: rook-ceph
-  pool: replicapool
-  imageFormat: "2"
-  imageFeatures: layering
-  csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
-  csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
-  csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
-  csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
-  csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
-  csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
-  csi.storage.k8s.io/fstype: ext4
-reclaimPolicy: Delete
-allowVolumeExpansion: true
---
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: rook-ceph-filesystem
-provisioner: rook-ceph.cephfs.csi.ceph.com
-parameters:
-  clusterID: rook-ceph
-  fsName: rook-shared-filesystem
-  pool: rook-shared-filesystem-data0
-  csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
-  csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
-  csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
-  csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
-  csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
-  csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
-reclaimPolicy: Delete
--- a/cluster/core/rook-ceph/storage/volumesnapshotclass.yaml
+++ b/cluster/core/rook-ceph/storage/volumesnapshotclass.yaml
@@ -1,16 +0,0 @@
---
-apiVersion: snapshot.storage.k8s.io/v1beta1
-kind: VolumeSnapshotClass
-metadata:
-  name: csi-rbdplugin-snapclass
-  annotations:
-    k10.kasten.io/is-snapshot-class: "true"
-driver: rook-ceph.rbd.csi.ceph.com
-parameters:
-  # Specify a string that identifies your cluster. Ceph CSI supports any
-  # unique string. When Ceph CSI is deployed by Rook use the Rook namespace,
-  # for example "rook-ceph".
-  clusterID: rook-ceph
-  csi.storage.k8s.io/snapshotter-secret-name: rook-csi-rbd-provisioner
-  csi.storage.k8s.io/snapshotter-secret-namespace: rook-ceph
-deletionPolicy: Delete