aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--cmd/seaweedfs-csi-driver/main.go40
-rw-r--r--deploy/helm/seaweedfs-csi-driver/templates/daemonset.yml11
-rw-r--r--deploy/helm/seaweedfs-csi-driver/templates/kubemod_modrule.yml29
-rw-r--r--deploy/helm/seaweedfs-csi-driver/values.yaml14
-rw-r--r--pkg/datalocality/mapping.go31
-rw-r--r--pkg/driver/driver.go3
-rw-r--r--pkg/driver/mounter_seaweedfs.go22
7 files changed, 149 insertions, 1 deletions
diff --git a/cmd/seaweedfs-csi-driver/main.go b/cmd/seaweedfs-csi-driver/main.go
index 99189d0..3a0f2c8 100644
--- a/cmd/seaweedfs-csi-driver/main.go
+++ b/cmd/seaweedfs-csi-driver/main.go
@@ -5,6 +5,7 @@ import (
"log"
"os"
+ "github.com/seaweedfs/seaweedfs-csi-driver/pkg/datalocality"
"github.com/seaweedfs/seaweedfs-csi-driver/pkg/driver"
"github.com/seaweedfs/seaweedfs/weed/glog"
flag "github.com/seaweedfs/seaweedfs/weed/util/fla9"
@@ -20,6 +21,9 @@ var (
cacheDir = flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data")
uidMap = flag.String("map.uid", "", "map local uid to uid on filer, comma-separated <local_uid>:<filer_uid>")
gidMap = flag.String("map.gid", "", "map local gid to gid on filer, comma-separated <local_gid>:<filer_gid>")
+ dataCenter = flag.String("dataCenter", "", "dataCenter this node is running in (locality-definition)")
+ dataLocalityStr = flag.String("dataLocality", "", "which volume-nodes pods will use for activity (one-of: 'write_preferLocalDc'). Requires used locality-definitions to be set")
+ dataLocality datalocality.DataLocality
)
func main() {
@@ -35,6 +39,18 @@ func main() {
os.Exit(0)
}
+ err := convertRequiredValues()
+ if(err != nil){
+ glog.Error("Failed converting flag: ", err);
+ os.Exit(1);
+ }
+
+ err = checkPreconditions()
+ if(err != nil){
+ glog.Error("Precondition failed: ", err);
+ os.Exit(1);
+ }
+
glog.Infof("connect to filer %s", *filer)
drv := driver.NewSeaweedFsDriver(*filer, *nodeID, *endpoint)
@@ -43,5 +59,29 @@ func main() {
drv.CacheDir = *cacheDir
drv.UidMap = *uidMap
drv.GidMap = *gidMap
+ drv.DataCenter = *dataCenter
+ drv.DataLocality = dataLocality
+
drv.Run()
}
+
+func convertRequiredValues() error {
+ // Convert DataLocalityStr to DataLocality
+ if(*dataLocalityStr != ""){
+ var ok bool
+ dataLocality, ok = datalocality.FromString(*dataLocalityStr)
+ if(!ok){
+ return fmt.Errorf("dataLocality invalid value")
+ }
+ }
+
+ return nil
+}
+
+func checkPreconditions() error {
+ if(dataLocality != datalocality.None && *dataCenter == ""){
+ return fmt.Errorf("dataLocality set, but not all locality-definitions were set! ('dataCenter')")
+ }
+
+ return nil
+}
diff --git a/deploy/helm/seaweedfs-csi-driver/templates/daemonset.yml b/deploy/helm/seaweedfs-csi-driver/templates/daemonset.yml
index f0596bb..b3b8401 100644
--- a/deploy/helm/seaweedfs-csi-driver/templates/daemonset.yml
+++ b/deploy/helm/seaweedfs-csi-driver/templates/daemonset.yml
@@ -59,6 +59,10 @@ spec:
- "--filer=$(SEAWEEDFS_FILER)"
- "--nodeid=$(NODE_ID)"
- "--cacheDir=/var/cache/seaweedfs"
+ - "--dataLocality={{ .Values.dataLocality }}"
+ {{- if .Values.node.injectTopologyInfoFromNodeLabel.enabled }}
+ - "--dataCenter=$(DATACENTER)"
+ {{- end }}
env:
- name: CSI_ENDPOINT
value: unix:///csi/csi.sock
@@ -68,6 +72,13 @@ spec:
valueFrom:
fieldRef:
fieldPath: spec.nodeName
+ {{- if .Values.node.injectTopologyInfoFromNodeLabel.enabled }}
+ - name: DATACENTER
+ valueFrom:
+ fieldRef:
+ # Injected by ModRule 'inject-topology-labels'
+ fieldPath: metadata.labels['{{ .Values.node.injectTopologyInfoFromNodeLabel.labels.dataCenter }}']
+ {{- end }}
{{- if .Values.tlsSecret }}
- name: WEED_GRPC_CLIENT_KEY
value: /var/run/secrets/app/tls/tls.key
diff --git a/deploy/helm/seaweedfs-csi-driver/templates/kubemod_modrule.yml b/deploy/helm/seaweedfs-csi-driver/templates/kubemod_modrule.yml
new file mode 100644
index 0000000..9e78b04
--- /dev/null
+++ b/deploy/helm/seaweedfs-csi-driver/templates/kubemod_modrule.yml
@@ -0,0 +1,29 @@
+# Based on https://github.com/kubernetes/kubernetes/issues/40610#issuecomment-1364368282
+{{- if .Values.node.injectTopologyInfoFromNodeLabel.enabled }}
+apiVersion: api.kubemod.io/v1beta1
+kind: ModRule
+metadata:
+ name: inject-topology-labels
+ #namespace: kubemod-system
+spec:
+ type: Patch
+ targetNamespaceRegex: ".*"
+ admissionOperations:
+ - UPDATE
+
+ match:
+ # Match pods...
+ - select: '{{ template "seaweedfs-csi-driver.name" . }}-node'
+ matchValue: 'Pod'
+ # ...which have access to the node's manifest through the synthetic ref injected by KubeMod.
+ - select: '$.syntheticRefs.node.metadata.labels'
+
+ patch:
+ # Grab the node's region and zone and put them in the pod's corresponding labels.
+ - op: add
+ path: /metadata/labels/topology.kubernetes.io~1region
+ value: '"{{ index .Target.syntheticRefs.node.metadata.labels "topology.kubernetes.io/region"}}"'
+ - op: add
+ path: /metadata/labels/topology.kubernetes.io~1zone
+ value: '"{{ index .Target.syntheticRefs.node.metadata.labels "topology.kubernetes.io/zone"}}"'
+{{- end }}
diff --git a/deploy/helm/seaweedfs-csi-driver/values.yaml b/deploy/helm/seaweedfs-csi-driver/values.yaml
index d76030e..bb29782 100644
--- a/deploy/helm/seaweedfs-csi-driver/values.yaml
+++ b/deploy/helm/seaweedfs-csi-driver/values.yaml
@@ -42,6 +42,12 @@ controller:
affinity: {}
tolerations: {}
+# DataLocality (inspired by Longhorn) allows instructing the storage-driver which volume-locations will be used or preferred in Pods to read & write.
+# e.g. Allows Pods to write preferrably to its local dataCenter volume-servers
+# Requires Volume-Servers to be correctly labelled and matching Topology-Info to be passed into seaweedfs-csi-driver node
+# Example-Value: "write_preferlocaldc"
+dataLocality: "none"
+
node:
# Deploy node daemonset
enabled: true
@@ -54,6 +60,14 @@ node:
affinity: {}
tolerations: {}
+ # Auto-Inject Topology-Info from Kubernetes node-labels using KubeMod (https://github.com/kubemod/kubemod)
+ # Necessary because DownwardAPI doesnt support passing node-labels (see: https://github.com/kubernetes/kubernetes/issues/40610)
+ # Requires KubeMod to be installed
+ injectTopologyInfoFromNodeLabel:
+ enabled: false
+ labels:
+ dataCenter: "topology.kubernetes.io/zone"
+
## Change if not using standard kubernetes deployments, like k0s
volumes:
registration_dir: /var/lib/kubelet/plugins_registry
diff --git a/pkg/datalocality/mapping.go b/pkg/datalocality/mapping.go
new file mode 100644
index 0000000..51484b0
--- /dev/null
+++ b/pkg/datalocality/mapping.go
@@ -0,0 +1,31 @@
+package datalocality
+
+import (
+ "strings"
+)
+
+type DataLocality uint
+
+const (
+ None DataLocality = iota
+ Write_preferLocalDc
+)
+
+// DataLocality -> String
+var dataLocalityStringMap = []string {
+ "none",
+ "write_preferlocaldc",
+}
+func (d DataLocality) String() string {
+ return dataLocalityStringMap[d]
+}
+
+// String -> DataLocality
+var stringDataLocalityMap = map[string]DataLocality {
+ "none": None,
+ "write_preferlocaldc": Write_preferLocalDc,
+}
+func FromString(s string) (DataLocality, bool) {
+ value, ok := stringDataLocalityMap[strings.ToLower(s)]
+ return value, ok
+}
diff --git a/pkg/driver/driver.go b/pkg/driver/driver.go
index 47a55e0..1cc3832 100644
--- a/pkg/driver/driver.go
+++ b/pkg/driver/driver.go
@@ -10,6 +10,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
+ "github.com/seaweedfs/seaweedfs-csi-driver/pkg/datalocality"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
@@ -43,6 +44,8 @@ type SeaweedFsDriver struct {
CacheDir string
UidMap string
GidMap string
+ DataCenter string
+ DataLocality datalocality.DataLocality
}
func NewSeaweedFsDriver(filer, nodeID, endpoint string) *SeaweedFsDriver {
diff --git a/pkg/driver/mounter_seaweedfs.go b/pkg/driver/mounter_seaweedfs.go
index 16d97c8..0fc4281 100644
--- a/pkg/driver/mounter_seaweedfs.go
+++ b/pkg/driver/mounter_seaweedfs.go
@@ -7,6 +7,7 @@ import (
"strconv"
"strings"
+ "github.com/seaweedfs/seaweedfs-csi-driver/pkg/datalocality"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/util"
)
@@ -93,6 +94,24 @@ func (seaweedFs *seaweedFsMounter) Mount(target string) (Unmounter, error) {
"readRetryTime": "",
}
+ // Handle DataLocality
+ dataLocality := seaweedFs.driver.DataLocality;
+ // Try to override when set in context
+ if dataLocalityStr, ok := seaweedFs.volContext["dataLocality"]; ok{
+ // Convert to enum
+ dataLocalityRes, ok := datalocality.FromString(dataLocalityStr)
+ if(!ok){
+ glog.Warning("volumeContext 'dataLocality' invalid");
+ }else{
+ dataLocality = dataLocalityRes
+ }
+ }
+ // Settings based on type
+ switch(dataLocality){
+ case datalocality.Write_preferLocalDc:
+ argsMap["dataCenter"] = seaweedFs.driver.DataCenter;
+ }
+
// volContext-parameter -> mount-arg
parameterArgMap := map[string]string{
"uidMap": "map.uid",
@@ -101,10 +120,11 @@ func (seaweedFs *seaweedFsMounter) Mount(target string) (Unmounter, error) {
// volumeContext has "diskType", but mount-option is "disk", converting for backwards compatability
"diskType": "disk",
}
-
+
// Explicitly ignored volContext args e.g. handled somewhere else
ignoreArgs := []string{
"volumeCapacity",
+ "dataLocality",
}
// Merge volContext into argsMap with key-mapping
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-21 16:18:33 +0000