Merge branch 'master' of https://github.com/seaweedfs/seaweedfs-csi-driver

9 files changed, 105 insertions, 37 deletions

diff --git a/.gitignore b/.gitignore
index d132cb8..1c659fb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,2 @@
-
+.vscode
 *.iml
diff --git a/README.md b/README.md
index 7897bd6..20ab423 100644
--- a/README.md
+++ b/README.md
@@ -44,7 +44,7 @@ $ kubectl apply -f deploy/kubernetes/sample-busybox-pod.yaml
 ```
 $ kubectl exec my-csi-app -- df -h
 ```
-8. Clean up 
+8. Clean up
 ```
 $ kubectl delete -f deploy/kubernetes/sample-busybox-pod.yaml
 $ kubectl delete -f deploy/kubernetes/sample-seaweedfs-pvc.yaml
@@ -53,7 +53,7 @@ $ kubectl delete -f deploy/kubernetes/seaweedfs-csi.yaml
 
 # Deployment by helm chart
 
-1. Clone project 
+1. Clone project
 ```bash
 git clone https://github.com/seaweedfs/seaweedfs-csi-driver.git
 ```
@@ -68,16 +68,17 @@ helm uninstall seaweedfs-csi-driver
 ```
 
 # Safe rollout update
-When update DaemonSet ( DS ) break processes who implements fuse mount. 
-And now new pod not remount net device
-
-For better safe update use ``node.updateStrategy.type: OnDelete`` in this need manual update. Steps:
- - delete DS pods on node where no exists seaweedfs PV
- - cordon or taint node 
- - evict or delete pods with seaweedfs PV
- - delete DS pod on node
- - uncordon or remove taint on node
- - repeat all steps on [all nodes 
+Updating seaweed-csi-driver DaemonSet (DS) will break processeses who implement fuse mount:
+newly created pods will not remount net device.
+
+For safe update set `node.updateStrategy.type: OnDelete` for manual update. Steps:
+
+  1. delete DS pods on the node where there is no seaweedfs PV
+  2. cordon or taint node
+  3. evict or delete pods with seaweedfs PV
+  4. delete DS pod on node
+  5. uncordon or remove taint on node
+  6. repeat all steps on [all nodes]
 
 # Static and dynamic provisioning
 
diff --git a/cmd/seaweedfs-csi-driver/main.go b/cmd/seaweedfs-csi-driver/main.go
index 76f5829..8eedce3 100644
--- a/cmd/seaweedfs-csi-driver/main.go
+++ b/cmd/seaweedfs-csi-driver/main.go
@@ -16,7 +16,7 @@ var (
 	nodeID            = flag.String("nodeid", "", "node id")
 	version           = flag.Bool("version", false, "Print the version and exit.")
 	concurrentWriters = flag.Int("concurrentWriters", 32, "limit concurrent goroutine writers if not 0")
-	cacheSizeMB       = flag.Int64("cacheCapacityMB", 0, "local file chunk cache capacity in MB")
+	cacheSizeMB       = flag.Int("cacheCapacityMB", 0, "local file chunk cache capacity in MB")
 	cacheDir          = flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data")
 	uidMap            = flag.String("map.uid", "", "map local uid to uid on filer, comma-separated <local_uid>:<filer_uid>")
 	gidMap            = flag.String("map.gid", "", "map local gid to gid on filer, comma-separated <local_gid>:<filer_gid>")
diff --git a/deploy/helm/seaweedfs-csi-driver/Chart.yaml b/deploy/helm/seaweedfs-csi-driver/Chart.yaml
index ac9c7aa..449ad42 100644
--- a/deploy/helm/seaweedfs-csi-driver/Chart.yaml
+++ b/deploy/helm/seaweedfs-csi-driver/Chart.yaml
@@ -2,5 +2,5 @@ apiVersion: v2
 name: seaweedfs-csi-driver
 description: A Helm chart for Kubernetes
 type: application
-version: 0.1.1
+version: 0.1.2
 appVersion: latest
diff --git a/deploy/helm/seaweedfs-csi-driver/templates/daemonset.yml b/deploy/helm/seaweedfs-csi-driver/templates/daemonset.yml
index 8e768d3..be31682 100644
--- a/deploy/helm/seaweedfs-csi-driver/templates/daemonset.yml
+++ b/deploy/helm/seaweedfs-csi-driver/templates/daemonset.yml
@@ -37,7 +37,7 @@ spec:
             - name: ADDRESS
               value: /csi/csi.sock
             - name: DRIVER_REG_SOCK_PATH
-              value: /var/lib/kubelet/plugins/{{ .Values.driverName }}/csi.sock
+              value: {{ .Values.node.volumes.plugins_dir }}/{{ .Values.driverName }}/csi.sock
             - name: KUBE_NODE_NAME
               valueFrom:
                 fieldRef:
@@ -61,6 +61,7 @@ spec:
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--filer=$(SEAWEEDFS_FILER)"
             - "--nodeid=$(NODE_ID)"
+            - "--cacheDir=/var/cache/seaweedfs"
           env:
             - name: CSI_ENDPOINT
               value: unix:///csi/csi.sock
@@ -88,10 +89,10 @@ spec:
             - name: plugin-dir
               mountPath: /csi
             - name: plugins-dir
-              mountPath: /var/lib/kubelet/plugins
+              mountPath: {{ .Values.node.volumes.plugins_dir }}
               mountPropagation: "Bidirectional"
             - name: pods-mount-dir
-              mountPath: /var/lib/kubelet/pods
+              mountPath: {{ .Values.node.volumes.pods_mount_dir }}
               mountPropagation: "Bidirectional"
             - mountPath: /dev
               name: device-dir
@@ -99,26 +100,30 @@ spec:
             - name: tls
               mountPath: /var/run/secrets/app/tls
             {{- end }}
+            - name: cache
+              mountPath: /var/cache/seaweedfs
       volumes:
         - name: registration-dir
           hostPath:
-            path: /var/lib/kubelet/plugins_registry/
+            path: {{ .Values.node.volumes.registration_dir }}
             type: DirectoryOrCreate
         - name: plugin-dir
           hostPath:
-            path: /var/lib/kubelet/plugins/{{ .Values.driverName }}
+            path: {{ .Values.node.volumes.plugins_dir }}/{{ .Values.driverName }}
             type: DirectoryOrCreate
         - name: plugins-dir
           hostPath:
-            path: /var/lib/kubelet/plugins
+            path: {{ .Values.node.volumes.plugins_dir }}
             type: Directory
         - name: pods-mount-dir
           hostPath:
-            path: /var/lib/kubelet/pods
+            path: {{ .Values.node.volumes.pods_mount_dir }}
             type: Directory
         - name: device-dir
           hostPath:
             path: /dev
+        - name: cache
+          emptyDir: {}
         {{- if .Values.tlsSecret }}
         - name: tls
           secret:
diff --git a/deploy/helm/seaweedfs-csi-driver/values.yaml b/deploy/helm/seaweedfs-csi-driver/values.yaml
index bb5af07..80637d6 100644
--- a/deploy/helm/seaweedfs-csi-driver/values.yaml
+++ b/deploy/helm/seaweedfs-csi-driver/values.yaml
@@ -30,7 +30,7 @@ seaweedfsCsiPlugin:
   image: chrislusf/seaweedfs-csi-driver:latest
   resources: {}
 
-# NOT Change, for future releases. Must be equal Name in GetPluginInfoResponse
+# DO NOT Change. Reserved for future releases. Must be equal Name in GetPluginInfoResponse
 driverName: seaweedfs-csi-driver
 
 controller:
@@ -40,11 +40,17 @@ controller:
 node:
   # Deploy node daemonset
   enabled: true
-  # When pod on node be recreated all pod on same node lost PV.
-  # For safe update use updateStrategy.type: OnDelete and manual move pods who use PV and delete damonset pod
+  # When seaweedfs-csi-driver-node pod on node is recreated, all pods on same node using seaweed-csi PV will stop working.
+  # For safe update set updateStrategy.type: OnDelete and manually move pods who use seaweed-csi PV, then delete  seaweedfs-csi-driver-node damonset pod
   updateStrategy:
     type: RollingUpdate
     rollingUpdate:
       maxUnavailable: 25%
   affinity: {}
   tolerations: {}
+
+  ## Change if not using standard kubernetes deployments, like k0s
+  volumes:
+    registration_dir: /var/lib/kubelet/plugins_registry
+    plugins_dir: /var/lib/kubelet/plugins
+    pods_mount_dir: /var/lib/kubelet/pods
diff --git a/pkg/driver/driver.go b/pkg/driver/driver.go
index 55cf0a3..00ef2d9 100644
--- a/pkg/driver/driver.go
+++ b/pkg/driver/driver.go
@@ -39,7 +39,7 @@ type SeaweedFsDriver struct {
 	filerIndex        int
 	grpcDialOption    grpc.DialOption
 	ConcurrentWriters int
-	CacheSizeMB       int64
+	CacheSizeMB       int
 	CacheDir          string
 	UidMap            string
 	GidMap            string
diff --git a/pkg/driver/mounter_seaweedfs.go b/pkg/driver/mounter_seaweedfs.go
index 5cbca71..a82b10b 100644
--- a/pkg/driver/mounter_seaweedfs.go
+++ b/pkg/driver/mounter_seaweedfs.go
@@ -2,6 +2,8 @@ package driver
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 
@@ -19,6 +21,11 @@ type seaweedFsMounter struct {
 	volContext map[string]string
 }
 
+type seaweedFsUnmounter struct {
+	unmounter Unmounter
+	cacheDir  string
+}
+
 const (
 	seaweedFsCmd = "weed"
 )
@@ -34,6 +41,25 @@ func newSeaweedFsMounter(volumeID string, path string, collection string, readOn
 	}, nil
 }
 
+func (seaweedFs *seaweedFsMounter) getOrDefaultContext(key string, defaultValue string) string {
+	v, ok := seaweedFs.volContext[key]
+	if ok {
+		return v
+	}
+	return defaultValue
+}
+
+func (seaweedFs *seaweedFsMounter) getOrDefaultContextInt(key string, defaultValue int) int {
+	v := seaweedFs.getOrDefaultContext(key, "")
+	if v != "" {
+		iv, err := strconv.Atoi(v)
+		if err == nil {
+			return iv
+		}
+	}
+	return defaultValue
+}
+
 func (seaweedFs *seaweedFsMounter) Mount(target string) (Unmounter, error) {
 	glog.V(0).Infof("mounting %v %s to %s", seaweedFs.driver.filers, seaweedFs.path, target)
 
@@ -51,7 +77,7 @@ func (seaweedFs *seaweedFsMounter) Mount(target string) (Unmounter, error) {
 		fmt.Sprintf("-collection=%s", seaweedFs.collection),
 		fmt.Sprintf("-filer=%s", strings.Join(filers, ",")),
 		fmt.Sprintf("-filer.path=%s", seaweedFs.path),
-		fmt.Sprintf("-cacheCapacityMB=%d", seaweedFs.driver.CacheSizeMB),
+		fmt.Sprintf("-cacheCapacityMB=%d", seaweedFs.getOrDefaultContextInt("cacheSizeMB", seaweedFs.driver.CacheSizeMB)),
 		fmt.Sprintf("-localSocket=%s", GetLocalSocket(seaweedFs.volumeID)),
 	}
 
@@ -78,24 +104,33 @@ func (seaweedFs *seaweedFsMounter) Mount(target string) (Unmounter, error) {
 		args = append(args, "-readOnly")
 	}
 
-	if seaweedFs.driver.ConcurrentWriters > 0 {
-		args = append(args, fmt.Sprintf("-concurrentWriters=%d", seaweedFs.driver.ConcurrentWriters))
-	}
-	if seaweedFs.driver.CacheDir != "" {
-		args = append(args, fmt.Sprintf("-cacheDir=%s", seaweedFs.driver.CacheDir))
+	// CacheDir should be always defined - we use temp dir in case it is not defined
+	// we need to use predictable cache path, because we need to clean it up on unstage
+	cacheDir := filepath.Join(seaweedFs.driver.CacheDir, seaweedFs.volumeID)
+	args = append(args, fmt.Sprintf("-cacheDir=%s", cacheDir))
+
+	if cw := seaweedFs.getOrDefaultContextInt("concurrentWriters", seaweedFs.driver.ConcurrentWriters); cw > 0 {
+		args = append(args, fmt.Sprintf("-concurrentWriters=%d", cw))
 	}
-	if seaweedFs.driver.UidMap != "" {
-		args = append(args, fmt.Sprintf("-map.uid=%s", seaweedFs.driver.UidMap))
+	if uidMap := seaweedFs.getOrDefaultContext("uidMap", seaweedFs.driver.UidMap); uidMap != "" {
+		args = append(args, fmt.Sprintf("-map.uid=%s", uidMap))
 	}
-	if seaweedFs.driver.GidMap != "" {
-		args = append(args, fmt.Sprintf("-map.gid=%s", seaweedFs.driver.GidMap))
+	if gidMap := seaweedFs.getOrDefaultContext("gidMap", seaweedFs.driver.GidMap); gidMap != "" {
+		args = append(args, fmt.Sprintf("-map.gid=%s", gidMap))
 	}
 
 	u, err := fuseMount(target, seaweedFsCmd, args)
 	if err != nil {
 		glog.Errorf("mount %v %s to %s: %s", seaweedFs.driver.filers, seaweedFs.path, target, err)
 	}
-	return u, err
+
+	return &seaweedFsUnmounter{unmounter: u, cacheDir: cacheDir}, err
+}
+
+func (su *seaweedFsUnmounter) Unmount() error {
+	err := su.unmounter.Unmount()
+	_ = os.RemoveAll(su.cacheDir)
+	return err
 }
 
 func GetLocalSocket(volumeID string) string {
diff --git a/pkg/driver/utils.go b/pkg/driver/utils.go
index 485d4da..b629543 100644
--- a/pkg/driver/utils.go
+++ b/pkg/driver/utils.go
@@ -3,6 +3,7 @@ package driver
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 
@@ -14,6 +15,10 @@ import (
 )
 
 func NewNodeServer(n *SeaweedFsDriver) *NodeServer {
+	if err := removeDirContent(n.CacheDir); err != nil {
+		glog.Warning("error cleaning up cache dir")
+	}
+
 	return &NodeServer{
 		Driver:        n,
 		volumeMutexes: NewKeyMutex(),
@@ -83,6 +88,22 @@ func checkMount(targetPath string) (bool, error) {
 	return notMnt, nil
 }
 
+func removeDirContent(path string) error {
+	files, err := filepath.Glob(filepath.Join(path, "*"))
+	if err != nil {
+		return err
+	}
+
+	for _, file := range files {
+		err = os.RemoveAll(file)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 type KeyMutex struct {
 	mutexes sync.Map
 }