Helm chart refactoring

* Separate NodeServer and ControllerServer
* Allow leader election for sidecars and replica count fot controller component
* Liveness probes
* Allow disabling attacher, cause this functionality is not used in reality

1 files changed, 260 insertions, 138 deletions

diff --git a/deploy/kubernetes/seaweedfs-csi.yaml b/deploy/kubernetes/seaweedfs-csi.yaml
index 276dcfd..6285419 100644
--- a/deploy/kubernetes/seaweedfs-csi.yaml
+++ b/deploy/kubernetes/seaweedfs-csi.yaml
@@ -1,17 +1,17 @@
 ---
-# Source: seaweedfs-csi-driver/templates/serviceaccounts.yml
+# Source: seaweedfs-csi-driver/templates/serviceaccounts.yaml
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: seaweedfs-controller-sa
 ---
-# Source: seaweedfs-csi-driver/templates/serviceaccounts.yml
+# Source: seaweedfs-csi-driver/templates/serviceaccounts.yaml
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: seaweedfs-node-sa
 ---
-# Source: seaweedfs-csi-driver/templates/storageclass.yml
+# Source: seaweedfs-csi-driver/templates/storageclass.yaml
 kind: StorageClass
 apiVersion: storage.k8s.io/v1
 metadata:
@@ -20,7 +20,7 @@ metadata:
 provisioner: seaweedfs-csi-driver
 allowVolumeExpansion: true
 ---
-# Source: seaweedfs-csi-driver/templates/rbac.yml
+# Source: seaweedfs-csi-driver/templates/rbac.yaml
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -54,7 +54,7 @@ rules:
     resources: [ "pods" ]
     verbs: [ "get", "list", "watch" ]
 ---
-# Source: seaweedfs-csi-driver/templates/rbac.yml
+# Source: seaweedfs-csi-driver/templates/rbac.yaml
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -73,41 +73,7 @@ rules:
     resources: ["volumeattachments", "volumeattachments/status"]
     verbs: ["get", "list", "watch", "update", "patch"]
 ---
-# Source: seaweedfs-csi-driver/templates/rbac.yml
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: seaweedfs-snapshotter-role
-rules:
-  - apiGroups: [""]
-    resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["persistentvolumeclaims"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["storageclasses"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["events"]
-    verbs: ["list", "watch", "create", "update", "patch"]
-  - apiGroups: [""]
-    resources: ["secrets"]
-    verbs: ["get", "list"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshotclasses"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshotcontents"]
-    verbs: ["create", "get", "list", "watch", "update", "delete"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshots"]
-    verbs: ["get", "list", "watch", "update"]
-  - apiGroups: ["apiextensions.k8s.io"]
-    resources: ["customresourcedefinitions"]
-    verbs: ["create", "list", "watch", "delete"]
----
-# Source: seaweedfs-csi-driver/templates/rbac.yml
+# Source: seaweedfs-csi-driver/templates/rbac.yaml
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -120,7 +86,7 @@ rules:
     resources: ["csinodes"]
     verbs: ["get", "list", "watch"]
 ---
-# Source: seaweedfs-csi-driver/templates/rbac.yml
+# Source: seaweedfs-csi-driver/templates/rbac.yaml
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -136,7 +102,7 @@ rules:
     resources: ["nodes"]
     verbs: ["get", "list", "watch"]
 ---
-# Source: seaweedfs-csi-driver/templates/rbac.yml
+# Source: seaweedfs-csi-driver/templates/rbac.yaml
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -144,12 +110,13 @@ metadata:
 subjects:
   - kind: ServiceAccount
     name: seaweedfs-controller-sa
+    namespace: default
 roleRef:
   kind: ClusterRole
   name: seaweedfs-provisioner-role
   apiGroup: rbac.authorization.k8s.io
 ---
-# Source: seaweedfs-csi-driver/templates/rbac.yml
+# Source: seaweedfs-csi-driver/templates/rbac.yaml
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -157,51 +124,67 @@ metadata:
 subjects:
   - kind: ServiceAccount
     name: seaweedfs-controller-sa
+    namespace: default
 roleRef:
   kind: ClusterRole
   name: seaweedfs-attacher-role
   apiGroup: rbac.authorization.k8s.io
 ---
-# Source: seaweedfs-csi-driver/templates/rbac.yml
+# Source: seaweedfs-csi-driver/templates/rbac.yaml
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: seaweedfs-snapshotter-binding
+  name: seaweedfs-driver-registrar-controller-binding
 subjects:
   - kind: ServiceAccount
     name: seaweedfs-controller-sa
+    namespace: default
 roleRef:
   kind: ClusterRole
-  name: seaweedfs-snapshotter-role
+  name: seaweedfs-driver-registrar-controller-role
   apiGroup: rbac.authorization.k8s.io
 ---
-# Source: seaweedfs-csi-driver/templates/rbac.yml
+# Source: seaweedfs-csi-driver/templates/rbac.yaml
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: seaweedfs-driver-registrar-controller-binding
+  name: seaweedfs-driver-registrar-node-binding
 subjects:
   - kind: ServiceAccount
-    name: seaweedfs-controller-sa
+    name: seaweedfs-node-sa
+    namespace: default
 roleRef:
   kind: ClusterRole
-  name: seaweedfs-driver-registrar-controller-role
+  name: seaweedfs-driver-registrar-node-role
   apiGroup: rbac.authorization.k8s.io
 ---
-# Source: seaweedfs-csi-driver/templates/rbac.yml
-kind: ClusterRoleBinding
+# Source: seaweedfs-csi-driver/templates/rbac.yaml
+kind: Role
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: seaweedfs-driver-registrar-node-binding
+  name: seaweedfs-leader-election-controller-role
+  namespace: default
+rules:
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
+---
+# Source: seaweedfs-csi-driver/templates/rbac.yaml
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: seaweedfs-leader-election-controller-binding
+  namespace: default
 subjects:
   - kind: ServiceAccount
-    name: seaweedfs-node-sa
+    namespace: default
+    name: seaweedfs-controller-sa
 roleRef:
-  kind: ClusterRole
-  name: seaweedfs-driver-registrar-node-role
+  kind: Role
+  name: seaweedfs-leader-election-controller-role
   apiGroup: rbac.authorization.k8s.io
 ---
-# Source: seaweedfs-csi-driver/templates/daemonset.yml
+# Source: seaweedfs-csi-driver/templates/daemonset.yaml
 kind: DaemonSet
 apiVersion: apps/v1
 metadata:
@@ -225,13 +208,68 @@ spec:
       #hostNetwork: true
       #dnsPolicy: ClusterFirstWithHostNet
       containers:
+        # SeaweedFs Plugin (node)
+        - name: csi-seaweedfs-plugin
+          securityContext: 
+            allowPrivilegeEscalation: true
+            capabilities:
+              add:
+              - SYS_ADMIN
+            privileged: true
+          image: chrislusf/seaweedfs-csi-driver:latest
+          imagePullPolicy: IfNotPresent
+          args:
+            - --endpoint=$(CSI_ENDPOINT)
+            - --filer=$(SEAWEEDFS_FILER)
+            - --nodeid=$(NODE_ID)
+            - --cacheDir=/var/cache/seaweedfs
+            - --dataLocality=none
+            - --node
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///csi/csi.sock
+            - name: SEAWEEDFS_FILER
+              value: "SEAWEEDFS_FILER:8888"
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          ports:
+            - containerPort: 9808
+              name: healthz
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 60
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi
+            - name: plugins-dir
+              mountPath: /var/lib/kubelet/plugins
+              mountPropagation: "Bidirectional"
+            - name: pods-mount-dir
+              mountPath: /var/lib/kubelet/pods
+              mountPropagation: "Bidirectional"
+            - mountPath: /dev
+              name: device-dir
+            - name: cache
+              mountPath: /var/cache/seaweedfs
+          resources: 
+            null
+
+        # driver registrar
         - name: driver-registrar
-          image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0
+          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0
           imagePullPolicy: IfNotPresent
           args:
-            - "--v=5"
-            - "--csi-address=$(ADDRESS)"
-            - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
+            - --csi-address=$(ADDRESS)
+            - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+            - --http-endpoint=:9809
+            #- --v=5
           env:
             - name: ADDRESS
               value: /csi/csi.sock
@@ -241,55 +279,56 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
-          resources:
-            
-            {}
+          ports:
+            - containerPort: 9809
+              name: healthz
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 60
           volumeMounts:
             - name: plugin-dir
               mountPath: /csi/
             - name: registration-dir
               mountPath: /registration/
-        - name: csi-seaweedfs-plugin
-          securityContext:
-            privileged: true
-            capabilities:
-              add: ["SYS_ADMIN"]
-            allowPrivilegeEscalation: true
-          image: chrislusf/seaweedfs-csi-driver:latest
+          resources: 
+            {}
+
+        # liveness probe
+        - name: csi-liveness-probe
+          image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0
           imagePullPolicy: IfNotPresent
-          args :
-            - "--endpoint=$(CSI_ENDPOINT)"
-            - "--filer=$(SEAWEEDFS_FILER)"
-            - "--nodeid=$(NODE_ID)"
+          args:
+            - --csi-address=$(ADDRESS)
+            - --http-endpoint=:9808
           env:
-            - name: CSI_ENDPOINT
-              value: unix:///csi/csi.sock
-            - name: SEAWEEDFS_FILER
-              value: "SEAWEEDFS_FILER:8888"
-            - name: NODE_ID
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-          resources:
-            
-            {}
+            - name: ADDRESS
+              value: /csi/csi.sock
+          ports:
+            - containerPort: 9808
+              name: livenessprobe
           volumeMounts:
             - name: plugin-dir
               mountPath: /csi
-            - name: pods-mount-dir
-              mountPath: /var/lib/kubelet/pods
-              mountPropagation: "Bidirectional"
-            - mountPath: /dev
-              name: device-dir
+          resources: 
+            {}
+
       volumes:
         - name: registration-dir
           hostPath:
-            path: /var/lib/kubelet/plugins_registry/
+            path: /var/lib/kubelet/plugins_registry
             type: DirectoryOrCreate
         - name: plugin-dir
           hostPath:
             path: /var/lib/kubelet/plugins/seaweedfs-csi-driver
             type: DirectoryOrCreate
+        - name: plugins-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins
+            type: Directory
         - name: pods-mount-dir
           hostPath:
             path: /var/lib/kubelet/pods
@@ -297,9 +336,11 @@ spec:
         - name: device-dir
           hostPath:
             path: /dev
+        - name: cache
+          emptyDir: {}
 ---
-# Source: seaweedfs-csi-driver/templates/statefulset.yml
-kind: StatefulSet
+# Source: seaweedfs-csi-driver/templates/deployment.yaml
+kind: Deployment
 apiVersion: apps/v1
 metadata:
   name: seaweedfs-controller
@@ -307,7 +348,6 @@ spec:
   selector:
     matchLabels:
       app: seaweedfs-controller
-  serviceName: "csi-seaweedfs"
   replicas: 1
   template:
     metadata:
@@ -316,83 +356,165 @@ spec:
     spec:
       priorityClassName: system-cluster-critical
       serviceAccountName: seaweedfs-controller-sa
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                      - seaweedfs-controller
+              topologyKey: kubernetes.io/hostname
       containers:
+        # SeaweedFs Plugin (controller)
+        - name: seaweedfs-csi-plugin
+          image: chrislusf/seaweedfs-csi-driver:latest
+          imagePullPolicy: IfNotPresent
+          args :
+            - --endpoint=$(CSI_ENDPOINT)
+            - --filer=$(SEAWEEDFS_FILER)
+            - --nodeid=$(NODE_ID)"
+            - --controller
+            - --attacher
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
+            - name: SEAWEEDFS_FILER
+              value: "SEAWEEDFS_FILER:8888"
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          ports:
+            - containerPort: 9808
+              name: healthz
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 60
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+          resources: 
+            {}
+
         # provisioner
         - name: csi-provisioner
-          image: k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0
+          image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0
+          imagePullPolicy: IfNotPresent
           args:
-            - "--csi-address=$(ADDRESS)"
-            - -v
-            - "9"
+            - --csi-address=$(ADDRESS)
+            - --leader-election
+            - --leader-election-namespace=default
+            - --http-endpoint=:9809
+            #- --v=9
           env:
             - name: ADDRESS
               value: /var/lib/csi/sockets/pluginproxy/csi.sock
-          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 9809
+              name: healthz
+          livenessProbe:
+            httpGet:
+              path: /healthz/leader-election
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 60
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
-          resources:
-            
+          resources: 
             {}
-        # attacher
-        - name: csi-attacher
-          image: k8s.gcr.io/sig-storage/csi-attacher:v3.4.0
+
+        # resizer
+        - name: csi-resizer
+          image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0
+          imagePullPolicy: IfNotPresent
           args:
-            - "--v=5"
-            - "--csi-address=$(ADDRESS)"
-            - "--timeout=120s"
+            - --csi-address=$(ADDRESS)
+            - --leader-election
+            - --leader-election-namespace=default
+            - --http-endpoint=:9810
+            #- --v=5
           env:
             - name: ADDRESS
               value: /var/lib/csi/sockets/pluginproxy/csi.sock
-          imagePullPolicy: IfNotPresent
-          resources:
-            
-            {}
+          ports:
+            - containerPort: 9810
+              name: healthz
+          livenessProbe:
+            httpGet:
+              path: /healthz/leader-election
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 60
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
-        # resizer
-        - name: csi-resizer
-          image: k8s.gcr.io/sig-storage/csi-resizer:v1.4.0
+          resources: 
+            {}
+        # attacher
+        - name: csi-attacher
+          image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0
+          imagePullPolicy: IfNotPresent
           args:
-            - "--v=5"
-            - "--csi-address=$(ADDRESS)"
-            - "--leader-election=false"
+            - --csi-address=$(ADDRESS)
+            - --leader-election
+            - --leader-election-namespace=default
+            - --http-endpoint=:9811
           env:
             - name: ADDRESS
               value: /var/lib/csi/sockets/pluginproxy/csi.sock
-          imagePullPolicy: IfNotPresent
-          resources:
-            
-            {}
+          ports:
+            - containerPort: 9811
+              name: healthz
+          livenessProbe:
+            httpGet:
+              path: /healthz/leader-election
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 60
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
-        # SeaweedFs Plugin
-        - name: seaweedfs-csi-plugin
-          image: chrislusf/seaweedfs-csi-driver:latest
+          resources: 
+            {}
+
+        # liveness probe
+        - name: csi-liveness-probe
+          image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0
           imagePullPolicy: IfNotPresent
-          args :
-            - "--endpoint=$(CSI_ENDPOINT)"
-            - "--filer=$(SEAWEEDFS_FILER)"
-            - "--nodeid=$(NODE_ID)"
+          args:
+            - --csi-address=$(ADDRESS)
+            - --http-endpoint=:9808
           env:
-            - name: CSI_ENDPOINT
-              value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
-            - name: SEAWEEDFS_FILER
-              value: "SEAWEEDFS_FILER:8888"
-            - name: NODE_ID
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          ports:
+            - containerPort: 9808
+              name: livenessprobe
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
+          resources: 
+            {}
+
       volumes:
         - name: socket-dir
           emptyDir: {}
 ---
-# Source: seaweedfs-csi-driver/templates/csidriver.yml
+# Source: seaweedfs-csi-driver/templates/kubemod_modrule.yaml
+# Based on https://github.com/kubernetes/kubernetes/issues/40610#issuecomment-1364368282
+---
+# Source: seaweedfs-csi-driver/templates/csidriver.yaml
 apiVersion: storage.k8s.io/v1
 kind: CSIDriver
 metadata: