diff options
| -rw-r--r-- | README.md | 58 | ||||
| -rw-r--r-- | cmd/seaweedfs-csi-driver/main.go | 40 | ||||
| -rw-r--r-- | deploy/helm/seaweedfs-csi-driver/templates/daemonset.yml | 16 | ||||
| -rw-r--r-- | deploy/helm/seaweedfs-csi-driver/templates/kubemod_modrule.yml | 28 | ||||
| -rw-r--r-- | deploy/helm/seaweedfs-csi-driver/values.yaml | 14 | ||||
| -rw-r--r-- | pkg/datalocality/mapping.go | 31 | ||||
| -rw-r--r-- | pkg/driver/driver.go | 3 | ||||
| -rw-r--r-- | pkg/driver/mounter_seaweedfs.go | 25 | ||||
| -rw-r--r-- | pkg/driver/utils.go | 8 |
9 files changed, 222 insertions, 1 deletions
@@ -187,6 +187,64 @@ spec: <br> +# DataLocality + +DataLocality (inspired by [Longhorn](https://longhorn.io/docs/latest/high-availability/data-locality/)) allows instructing the storage-driver which volume-locations will be used or preferred in Pods to read & write. + +It auto-sets mount-options based on the location a pod is scheduled in and the locality-option wanted. +The option can be set and overridden in *Driver*, *StorageClass* and *PersistentVolume*. + +<br> + +## Setup + +Change the type of locality + +Level | Location +------------------- | -------- +Driver | Helm: `values.yaml` -> `dataLocality` <br> Or `DaemonSet` -> Container `csi-seaweedfs-plugin` -> args `--dataLocality=` +StorageClass | `parameter.dataLocality` +PersistentVolume | `spec.csi.volumeAttributes.dataLocality` + +Driver < StorageClass < PersistentVolume + +<br> + +## Available options + +Option | Effect +----------------------- | ------ +`none`* | Changes nothing +`write_preferLocalDc` | Sets the `DataCenter`-mount-option to the current Node-DataCenter, making writes local and allowing reads to occur wherever read data is stored. [More Details](#`write_preferLocalDc`) + +\* Default + +<br> + +## Requirements + +Volume-Servers and the CSI-Driver-Node need to have the locality-option `DataCenter` correctly set (currently only this option is required). + +This can be done manually (although quite tedious) or injected by the Container-Orchestration. + +<br> + +### Automatic injection + +**Kubernetes** + +Unfortunately Kubernetes doesnt allow grabbing node-labels, which contain well-known region-labels, and setting them as environment-variables. +The DownwardAPI is very limited in that regard. (see [#40610](https://github.com/kubernetes/kubernetes/issues/40610)) + +Therefore a workaround must be used. [KubeMod](https://github.com/kubemod/kubemod) can be used based on [this comment](https://github.com/kubernetes/kubernetes/issues/40610#issuecomment-1364368282). This of course requires KubeMod to be installed. + +You can activate it in the Helm-Chart `values.yaml` -> `node.injectTopologyInfoFromNodeLabel.enabled`. +`node.injectTopologyInfoFromNodeLabel.labels` decides which labels are grabbed from the node. + +It is recommended to use [well-known labels](https://kubernetes.io/docs/reference/labels-annotations-taints/#topologykubernetesioregion) to avoid confusion. + +<br> + # License [Apache v2 license](https://www.apache.org/licenses/LICENSE-2.0) diff --git a/cmd/seaweedfs-csi-driver/main.go b/cmd/seaweedfs-csi-driver/main.go index 99189d0..5542b93 100644 --- a/cmd/seaweedfs-csi-driver/main.go +++ b/cmd/seaweedfs-csi-driver/main.go @@ -5,6 +5,7 @@ import ( "log" "os" + "github.com/seaweedfs/seaweedfs-csi-driver/pkg/datalocality" "github.com/seaweedfs/seaweedfs-csi-driver/pkg/driver" "github.com/seaweedfs/seaweedfs/weed/glog" flag "github.com/seaweedfs/seaweedfs/weed/util/fla9" @@ -20,6 +21,9 @@ var ( cacheDir = flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data") uidMap = flag.String("map.uid", "", "map local uid to uid on filer, comma-separated <local_uid>:<filer_uid>") gidMap = flag.String("map.gid", "", "map local gid to gid on filer, comma-separated <local_gid>:<filer_gid>") + dataCenter = flag.String("dataCenter", "", "dataCenter this node is running in (locality-definition)") + dataLocalityStr = flag.String("dataLocality", "", "which volume-nodes pods will use for activity (one-of: 'write_preferLocalDc'). Requires used locality-definitions to be set") + dataLocality datalocality.DataLocality ) func main() { @@ -35,6 +39,18 @@ func main() { os.Exit(0) } + err := convertRequiredValues() + if(err != nil){ + glog.Error("Failed converting flag: ", err); + os.Exit(1); + } + + err = checkPreconditions() + if(err != nil){ + glog.Error("Precondition failed: ", err); + os.Exit(1); + } + glog.Infof("connect to filer %s", *filer) drv := driver.NewSeaweedFsDriver(*filer, *nodeID, *endpoint) @@ -43,5 +59,29 @@ func main() { drv.CacheDir = *cacheDir drv.UidMap = *uidMap drv.GidMap = *gidMap + drv.DataCenter = *dataCenter + drv.DataLocality = dataLocality + drv.Run() } + +func convertRequiredValues() error { + // Convert DataLocalityStr to DataLocality + if(*dataLocalityStr != ""){ + var ok bool + dataLocality, ok = datalocality.FromString(*dataLocalityStr) + if(!ok){ + return fmt.Errorf("dataLocality invalid value") + } + } + + return nil +} + +func checkPreconditions() error { + if err := driver.CheckDataLocality(&dataLocality, dataCenter); err != nil { + return err + } + + return nil +} diff --git a/deploy/helm/seaweedfs-csi-driver/templates/daemonset.yml b/deploy/helm/seaweedfs-csi-driver/templates/daemonset.yml index f0596bb..8d90cd3 100644 --- a/deploy/helm/seaweedfs-csi-driver/templates/daemonset.yml +++ b/deploy/helm/seaweedfs-csi-driver/templates/daemonset.yml @@ -14,6 +14,11 @@ spec: metadata: labels: app: {{ template "seaweedfs-csi-driver.name" . }}-node + {{- if .Values.node.injectTopologyInfoFromNodeLabel.enabled }} + annotations: + # Tell KubeMod to make node metadata available to pod ModRules. + ref.kubemod.io/inject-node-ref: "true" + {{- end }} spec: priorityClassName: system-node-critical serviceAccountName: {{ template "seaweedfs-csi-driver.name" . }}-node-sa @@ -59,6 +64,10 @@ spec: - "--filer=$(SEAWEEDFS_FILER)" - "--nodeid=$(NODE_ID)" - "--cacheDir=/var/cache/seaweedfs" + - "--dataLocality={{ .Values.dataLocality }}" + {{- if .Values.node.injectTopologyInfoFromNodeLabel.enabled }} + - "--dataCenter=$(DATACENTER)" + {{- end }} env: - name: CSI_ENDPOINT value: unix:///csi/csi.sock @@ -68,6 +77,13 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + {{- if .Values.node.injectTopologyInfoFromNodeLabel.enabled }} + - name: DATACENTER + valueFrom: + fieldRef: + # Injected by ModRule 'inject-topology-labels' + fieldPath: metadata.labels['dataCenter'] + {{- end }} {{- if .Values.tlsSecret }} - name: WEED_GRPC_CLIENT_KEY value: /var/run/secrets/app/tls/tls.key diff --git a/deploy/helm/seaweedfs-csi-driver/templates/kubemod_modrule.yml b/deploy/helm/seaweedfs-csi-driver/templates/kubemod_modrule.yml new file mode 100644 index 0000000..af8cf39 --- /dev/null +++ b/deploy/helm/seaweedfs-csi-driver/templates/kubemod_modrule.yml @@ -0,0 +1,28 @@ +# Based on https://github.com/kubernetes/kubernetes/issues/40610#issuecomment-1364368282 +{{- if .Values.node.injectTopologyInfoFromNodeLabel.enabled }} +apiVersion: api.kubemod.io/v1beta1 +kind: ModRule +metadata: + name: inject-topology-labels +spec: + type: Patch + targetNamespaceRegex: ".*" + admissionOperations: + - UPDATE + + match: + # Match pods... + - select: '$.kind' + matchValue: 'Pod' + # ... with label app = seaweedfs-csi-driver ... + - select: '$.metadata.labels.app' + matchValue: '{{ template "seaweedfs-csi-driver.name" . }}-node' + # ...which have access to the node's manifest through the synthetic ref injected by KubeMod. + - select: '$.syntheticRefs.node.metadata.labels' + + patch: + # Grab the node's region and zone and put them in the pod's corresponding labels. + - op: add + path: /metadata/labels/dataCenter + value: '{{`{{`}} index .Target.syntheticRefs.node.metadata.labels "{{ .Values.node.injectTopologyInfoFromNodeLabel.labels.dataCenter }}" {{`}}`}}' +{{- end }} diff --git a/deploy/helm/seaweedfs-csi-driver/values.yaml b/deploy/helm/seaweedfs-csi-driver/values.yaml index d76030e..bb29782 100644 --- a/deploy/helm/seaweedfs-csi-driver/values.yaml +++ b/deploy/helm/seaweedfs-csi-driver/values.yaml @@ -42,6 +42,12 @@ controller: affinity: {} tolerations: {} +# DataLocality (inspired by Longhorn) allows instructing the storage-driver which volume-locations will be used or preferred in Pods to read & write. +# e.g. Allows Pods to write preferrably to its local dataCenter volume-servers +# Requires Volume-Servers to be correctly labelled and matching Topology-Info to be passed into seaweedfs-csi-driver node +# Example-Value: "write_preferlocaldc" +dataLocality: "none" + node: # Deploy node daemonset enabled: true @@ -54,6 +60,14 @@ node: affinity: {} tolerations: {} + # Auto-Inject Topology-Info from Kubernetes node-labels using KubeMod (https://github.com/kubemod/kubemod) + # Necessary because DownwardAPI doesnt support passing node-labels (see: https://github.com/kubernetes/kubernetes/issues/40610) + # Requires KubeMod to be installed + injectTopologyInfoFromNodeLabel: + enabled: false + labels: + dataCenter: "topology.kubernetes.io/zone" + ## Change if not using standard kubernetes deployments, like k0s volumes: registration_dir: /var/lib/kubelet/plugins_registry diff --git a/pkg/datalocality/mapping.go b/pkg/datalocality/mapping.go new file mode 100644 index 0000000..51484b0 --- /dev/null +++ b/pkg/datalocality/mapping.go @@ -0,0 +1,31 @@ +package datalocality + +import ( + "strings" +) + +type DataLocality uint + +const ( + None DataLocality = iota + Write_preferLocalDc +) + +// DataLocality -> String +var dataLocalityStringMap = []string { + "none", + "write_preferlocaldc", +} +func (d DataLocality) String() string { + return dataLocalityStringMap[d] +} + +// String -> DataLocality +var stringDataLocalityMap = map[string]DataLocality { + "none": None, + "write_preferlocaldc": Write_preferLocalDc, +} +func FromString(s string) (DataLocality, bool) { + value, ok := stringDataLocalityMap[strings.ToLower(s)] + return value, ok +} diff --git a/pkg/driver/driver.go b/pkg/driver/driver.go index 47a55e0..1cc3832 100644 --- a/pkg/driver/driver.go +++ b/pkg/driver/driver.go @@ -10,6 +10,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/util" + "github.com/seaweedfs/seaweedfs-csi-driver/pkg/datalocality" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -43,6 +44,8 @@ type SeaweedFsDriver struct { CacheDir string UidMap string GidMap string + DataCenter string + DataLocality datalocality.DataLocality } func NewSeaweedFsDriver(filer, nodeID, endpoint string) *SeaweedFsDriver { diff --git a/pkg/driver/mounter_seaweedfs.go b/pkg/driver/mounter_seaweedfs.go index 16d97c8..12142af 100644 --- a/pkg/driver/mounter_seaweedfs.go +++ b/pkg/driver/mounter_seaweedfs.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" + "github.com/seaweedfs/seaweedfs-csi-driver/pkg/datalocality" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/util" ) @@ -93,6 +94,27 @@ func (seaweedFs *seaweedFsMounter) Mount(target string) (Unmounter, error) { "readRetryTime": "", } + // Handle DataLocality + dataLocality := seaweedFs.driver.DataLocality; + // Try to override when set in context + if dataLocalityStr, ok := seaweedFs.volContext["dataLocality"]; ok{ + // Convert to enum + dataLocalityRes, ok := datalocality.FromString(dataLocalityStr) + if(!ok){ + glog.Warning("volumeContext 'dataLocality' invalid"); + }else{ + dataLocality = dataLocalityRes + } + } + if err := CheckDataLocality(&dataLocality, &seaweedFs.driver.DataCenter); err != nil { + return nil, err + } + // Settings based on type + switch(dataLocality){ + case datalocality.Write_preferLocalDc: + argsMap["dataCenter"] = seaweedFs.driver.DataCenter; + } + // volContext-parameter -> mount-arg parameterArgMap := map[string]string{ "uidMap": "map.uid", @@ -101,10 +123,11 @@ func (seaweedFs *seaweedFsMounter) Mount(target string) (Unmounter, error) { // volumeContext has "diskType", but mount-option is "disk", converting for backwards compatability "diskType": "disk", } - + // Explicitly ignored volContext args e.g. handled somewhere else ignoreArgs := []string{ "volumeCapacity", + "dataLocality", } // Merge volContext into argsMap with key-mapping diff --git a/pkg/driver/utils.go b/pkg/driver/utils.go index b629543..e103079 100644 --- a/pkg/driver/utils.go +++ b/pkg/driver/utils.go @@ -8,6 +8,7 @@ import ( "sync" "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/seaweedfs/seaweedfs-csi-driver/pkg/datalocality" "github.com/seaweedfs/seaweedfs/weed/glog" "golang.org/x/net/context" "google.golang.org/grpc" @@ -121,3 +122,10 @@ func (km *KeyMutex) GetMutex(key string) *sync.Mutex { func (km *KeyMutex) RemoveMutex(key string) { km.mutexes.Delete(key) } + +func CheckDataLocality(dataLocality *datalocality.DataLocality, dataCenter *string) error { + if(*dataLocality != datalocality.None && *dataCenter == ""){ + return fmt.Errorf("dataLocality set, but not all locality-definitions were set") + } + return nil +} |