diff options
112 files changed, 5589 insertions, 957 deletions
diff --git a/docker/Dockerfile.rocksdb_large b/docker/Dockerfile.rocksdb_large index b5d569f44..af6068103 100644 --- a/docker/Dockerfile.rocksdb_large +++ b/docker/Dockerfile.rocksdb_large @@ -9,7 +9,7 @@ ENV ROCKSDB_VERSION v6.22.1 RUN cd /tmp && \ git clone https://github.com/facebook/rocksdb.git /tmp/rocksdb --depth 1 --single-branch --branch $ROCKSDB_VERSION && \ cd rocksdb && \ - make static_lib && \ + PORTABLE=1 make static_lib && \ make install-static ENV CGO_CFLAGS "-I/tmp/rocksdb/include" @@ -29,7 +29,7 @@ FROM alpine AS final LABEL author="Chris Lu" COPY --from=builder /go/bin/weed /usr/bin/ RUN mkdir -p /etc/seaweedfs -COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml +COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer_rocksdb.toml /etc/seaweedfs/filer.toml COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh RUN apk add fuse snappy gflags @@ -50,7 +50,7 @@ EXPOSE 8333 # webdav server http port EXPOSE 7333 -RUN mkdir -p /data/filerldb2 +RUN mkdir -p /data/filer_rocksdb VOLUME /data diff --git a/docker/filer_rocksdb.toml b/docker/filer_rocksdb.toml new file mode 100644 index 000000000..c1c74a64e --- /dev/null +++ b/docker/filer_rocksdb.toml @@ -0,0 +1,3 @@ +[rocksdb] +enabled = true +dir = "/data/filer_rocksdb" @@ -162,7 +162,10 @@ require ( modernc.org/token v1.0.0 // indirect ) -require github.com/fluent/fluent-logger-golang v1.8.0 +require ( + github.com/fluent/fluent-logger-golang v1.8.0 + github.com/hanwen/go-fuse/v2 v2.1.0 +) require ( cloud.google.com/go/kms v1.0.0 // indirect @@ -415,6 +415,10 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= +github.com/hanwen/go-fuse v1.0.0 h1:GxS9Zrn6c35/BnfiVsZVWmsG803xwE7eVRDvcf/BEVc= +github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok= +github.com/hanwen/go-fuse/v2 v2.1.0 h1:+32ffteETaLYClUj0a3aHjZ1hOPxxaNEHiZiujuDaek= +github.com/hanwen/go-fuse/v2 v2.1.0/go.mod h1:oRyA5eK+pvJyv5otpO/DgccS8y/RvYMaO00GgRLGryc= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -512,6 +516,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kurin/blazer v0.5.3 h1:SAgYv0TKU0kN/ETfO5ExjNAPyMt2FocO2s/UlCHfjAk= github.com/kurin/blazer v0.5.3/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU= +github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= +github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E= github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= diff --git a/k8s/helm_charts2/Chart.yaml b/k8s/helm_charts2/Chart.yaml index 6e92be7d8..93ea003bf 100644 --- a/k8s/helm_charts2/Chart.yaml +++ b/k8s/helm_charts2/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 description: SeaweedFS name: seaweedfs -appVersion: "2.88" -version: "2.88" +appVersion: "2.89" +version: "2.89" diff --git a/k8s/helm_charts2/templates/volume-statefulset.yaml b/k8s/helm_charts2/templates/volume-statefulset.yaml index 2168b2b80..de2703d14 100644 --- a/k8s/helm_charts2/templates/volume-statefulset.yaml +++ b/k8s/helm_charts2/templates/volume-statefulset.yaml @@ -40,7 +40,7 @@ spec: imagePullSecrets: - name: {{ .Values.global.imagePullSecrets }} {{- end }} - terminationGracePeriodSeconds: 10 + terminationGracePeriodSeconds: 150 {{- if .Values.volume.priorityClassName }} priorityClassName: {{ .Values.volume.priorityClassName | quote }} {{- end }} diff --git a/other/java/examples/pom.xml b/other/java/examples/pom.xml index 3c02bdfab..c0927559a 100644 --- a/other/java/examples/pom.xml +++ b/other/java/examples/pom.xml @@ -23,7 +23,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> - <version>2.9.2</version> + <version>2.10.1</version> <scope>compile</scope> </dependency> </dependencies> diff --git a/other/java/hdfs-over-ftp/pom.xml b/other/java/hdfs-over-ftp/pom.xml index 0db422db5..3cae6437a 100644 --- a/other/java/hdfs-over-ftp/pom.xml +++ b/other/java/hdfs-over-ftp/pom.xml @@ -36,7 +36,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> - <version>3.2.1</version> + <version>3.2.2</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> diff --git a/other/java/hdfs2/pom.xml b/other/java/hdfs2/pom.xml index fc49fe946..eccbb54bf 100644 --- a/other/java/hdfs2/pom.xml +++ b/other/java/hdfs2/pom.xml @@ -6,7 +6,7 @@ <properties> <seaweedfs.client.version>2.85</seaweedfs.client.version> - <hadoop.version>2.9.2</hadoop.version> + <hadoop.version>2.10.1</hadoop.version> </properties> <groupId>com.github.chrislusf</groupId> diff --git a/other/java/hdfs3/pom.xml b/other/java/hdfs3/pom.xml index 352174732..345d19c0c 100644 --- a/other/java/hdfs3/pom.xml +++ b/other/java/hdfs3/pom.xml @@ -6,7 +6,7 @@ <properties> <seaweedfs.client.version>2.85</seaweedfs.client.version> - <hadoop.version>3.1.1</hadoop.version> + <hadoop.version>3.1.4</hadoop.version> </properties> <groupId>com.github.chrislusf</groupId> diff --git a/weed/Makefile b/weed/Makefile index 4e871a71e..1d1a8476c 100644 --- a/weed/Makefile +++ b/weed/Makefile @@ -21,6 +21,10 @@ debug_mount: go build -gcflags="all=-N -l" dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 mount -dir=~/tmp/mm -cacheCapacityMB=0 -filer.path=/ -umask=000 +debug_mount2: + go build -gcflags="all=-N -l" + dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 mount2 -dir=~/tmp/mm -cacheCapacityMB=0 -filer.path=/ -umask=000 + debug_server: go build -gcflags="all=-N -l" dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- server -dir=~/tmp/99 -filer -volume.port=8343 -s3 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 diff --git a/weed/command/command.go b/weed/command/command.go index dbc18a053..c6665a7be 100644 --- a/weed/command/command.go +++ b/weed/command/command.go @@ -30,6 +30,7 @@ var Commands = []*Command{ cmdMaster, cmdMasterFollower, cmdMount, + cmdMount2, cmdS3, cmdIam, cmdMsgBroker, diff --git a/weed/command/filer.go b/weed/command/filer.go index 876b1bbf0..f886f1258 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -96,7 +96,7 @@ func init() { filerWebDavOptions.tlsPrivateKey = cmdFiler.Flag.String("webdav.key.file", "", "path to the TLS private key file") filerWebDavOptions.tlsCertificate = cmdFiler.Flag.String("webdav.cert.file", "", "path to the TLS certificate file") filerWebDavOptions.cacheDir = cmdFiler.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks") - filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 1000, "local cache capacity in MB") + filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB") // start iam on filer filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service") diff --git a/weed/command/filer_cat.go b/weed/command/filer_cat.go index 7f613f72b..ada843dea 100644 --- a/weed/command/filer_cat.go +++ b/weed/command/filer_cat.go @@ -8,7 +8,6 @@ import ( "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/wdclient" "google.golang.org/grpc" - "math" "net/url" "os" "strings" @@ -115,7 +114,7 @@ func runFilerCat(cmd *Command, args []string) bool { filerCat.filerClient = client - return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64) + return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, int64(filer.FileSize(respLookupEntry.Entry))) }) diff --git a/weed/command/master.go b/weed/command/master.go index 0f598f2da..3e37f827b 100644 --- a/weed/command/master.go +++ b/weed/command/master.go @@ -44,6 +44,8 @@ type MasterOptions struct { metricsIntervalSec *int raftResumeState *bool metricsHttpPort *int + heartbeatInterval *time.Duration + electionTimeout *time.Duration } func init() { @@ -65,6 +67,8 @@ func init() { m.metricsIntervalSec = cmdMaster.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds") m.metricsHttpPort = cmdMaster.Flag.Int("metricsPort", 0, "Prometheus metrics listen port") m.raftResumeState = cmdMaster.Flag.Bool("resumeState", false, "resume previous state on start master server") + m.heartbeatInterval = cmdMaster.Flag.Duration("heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)") + m.electionTimeout = cmdMaster.Flag.Duration("electionTimeout", 10*time.Second, "election timeout of master servers") } var cmdMaster = &Command{ @@ -132,8 +136,17 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) { glog.Fatalf("Master startup error: %v", e) } // start raftServer - raftServer, err := weed_server.NewRaftServer(security.LoadClientTLS(util.GetViper(), "grpc.master"), - peers, myMasterAddress, util.ResolvePath(*masterOption.metaFolder), ms.Topo, *masterOption.raftResumeState) + raftServerOption := &weed_server.RaftServerOption{ + GrpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.master"), + Peers: peers, + ServerAddr: myMasterAddress, + DataDir: util.ResolvePath(*masterOption.metaFolder), + Topo: ms.Topo, + RaftResumeState: *masterOption.raftResumeState, + HeartbeatInterval: *masterOption.heartbeatInterval, + ElectionTimeout: *masterOption.electionTimeout, + } + raftServer, err := weed_server.NewRaftServer(raftServerOption) if raftServer == nil { glog.Fatalf("please verify %s is writable, see https://github.com/chrislusf/seaweedfs/issues/717: %s", *masterOption.metaFolder, err) } diff --git a/weed/command/mount.go b/weed/command/mount.go index e54f1f07f..545ba8a43 100644 --- a/weed/command/mount.go +++ b/weed/command/mount.go @@ -50,7 +50,7 @@ func init() { mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 2, "local write buffer size, also chunk large files") mountOptions.concurrentWriters = cmdMount.Flag.Int("concurrentWriters", 32, "limit concurrent goroutine writers if not 0") mountOptions.cacheDir = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data") - mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 1000, "local file chunk cache capacity in MB (0 will disable cache)") + mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 0, "local file chunk cache capacity in MB") mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center") mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system") mountOptions.umaskString = cmdMount.Flag.String("umask", "022", "octal umask, e.g., 022, 0111") diff --git a/weed/command/mount2.go b/weed/command/mount2.go new file mode 100644 index 000000000..b285f5d3f --- /dev/null +++ b/weed/command/mount2.go @@ -0,0 +1,83 @@ +package command + +import ( + "os" + "time" +) + +type Mount2Options struct { + filer *string + filerMountRootPath *string + dir *string + dirAutoCreate *bool + collection *string + replication *string + diskType *string + ttlSec *int + chunkSizeLimitMB *int + concurrentWriters *int + cacheDir *string + cacheSizeMB *int64 + dataCenter *string + allowOthers *bool + umaskString *string + nonempty *bool + volumeServerAccess *string + uidMap *string + gidMap *string + readOnly *bool + debug *bool + debugPort *int +} + +var ( + mount2Options Mount2Options +) + +func init() { + cmdMount2.Run = runMount2 // break init cycle + mount2Options.filer = cmdMount2.Flag.String("filer", "localhost:8888", "comma-separated weed filer location") + mount2Options.filerMountRootPath = cmdMount2.Flag.String("filer.path", "/", "mount this remote path from filer server") + mount2Options.dir = cmdMount2.Flag.String("dir", ".", "mount weed filer to this directory") + mount2Options.dirAutoCreate = cmdMount2.Flag.Bool("dirAutoCreate", false, "auto create the directory to mount to") + mount2Options.collection = cmdMount2.Flag.String("collection", "", "collection to create the files") + mount2Options.replication = cmdMount2.Flag.String("replication", "", "replication(e.g. 000, 001) to create to files. If empty, let filer decide.") + mount2Options.diskType = cmdMount2.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag") + mount2Options.ttlSec = cmdMount2.Flag.Int("ttl", 0, "file ttl in seconds") + mount2Options.chunkSizeLimitMB = cmdMount2.Flag.Int("chunkSizeLimitMB", 2, "local write buffer size, also chunk large files") + mount2Options.concurrentWriters = cmdMount2.Flag.Int("concurrentWriters", 32, "limit concurrent goroutine writers if not 0") + mount2Options.cacheDir = cmdMount2.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks and meta data") + mount2Options.cacheSizeMB = cmdMount2.Flag.Int64("cacheCapacityMB", 0, "local file chunk cache capacity in MB") + mount2Options.dataCenter = cmdMount2.Flag.String("dataCenter", "", "prefer to write to the data center") + mount2Options.allowOthers = cmdMount2.Flag.Bool("allowOthers", true, "allows other users to access the file system") + mount2Options.umaskString = cmdMount2.Flag.String("umask", "022", "octal umask, e.g., 022, 0111") + mount2Options.nonempty = cmdMount2.Flag.Bool("nonempty", false, "allows the mounting over a non-empty directory") + mount2Options.volumeServerAccess = cmdMount2.Flag.String("volumeServerAccess", "direct", "access volume servers by [direct|publicUrl|filerProxy]") + mount2Options.uidMap = cmdMount2.Flag.String("map.uid", "", "map local uid to uid on filer, comma-separated <local_uid>:<filer_uid>") + mount2Options.gidMap = cmdMount2.Flag.String("map.gid", "", "map local gid to gid on filer, comma-separated <local_gid>:<filer_gid>") + mount2Options.readOnly = cmdMount2.Flag.Bool("readOnly", false, "read only") + mount2Options.debug = cmdMount2.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:<debug.port>/debug/pprof/goroutine?debug=2") + mount2Options.debugPort = cmdMount2.Flag.Int("debug.port", 6061, "http port for debugging") + + mountCpuProfile = cmdMount2.Flag.String("cpuprofile", "", "cpu profile output file") + mountMemProfile = cmdMount2.Flag.String("memprofile", "", "memory profile output file") + mountReadRetryTime = cmdMount2.Flag.Duration("readRetryTime", 6*time.Second, "maximum read retry wait time") +} + +var cmdMount2 = &Command{ + UsageLine: "mount2 -filer=localhost:8888 -dir=/some/dir", + Short: "<WIP> mount weed filer to a directory as file system in userspace(FUSE)", + Long: `mount weed filer to userspace. + + Pre-requisites: + 1) have SeaweedFS master and volume servers running + 2) have a "weed filer" running + These 2 requirements can be achieved with one command "weed server -filer=true" + + This uses github.com/seaweedfs/fuse, which enables writing FUSE file systems on + Linux, and OS X. + + On OS X, it requires OSXFUSE (http://osxfuse.github.com/). + + `, +} diff --git a/weed/command/mount2_notsupported.go b/weed/command/mount2_notsupported.go new file mode 100644 index 000000000..075b73436 --- /dev/null +++ b/weed/command/mount2_notsupported.go @@ -0,0 +1,15 @@ +//go:build !linux && !darwin +// +build !linux,!darwin + +package command + +import ( + "fmt" + "runtime" +) + +func runMount2(cmd *Command, args []string) bool { + fmt.Printf("Mount is not supported on %s %s\n", runtime.GOOS, runtime.GOARCH) + + return true +} diff --git a/weed/command/mount2_std.go b/weed/command/mount2_std.go new file mode 100644 index 000000000..89935e684 --- /dev/null +++ b/weed/command/mount2_std.go @@ -0,0 +1,213 @@ +//go:build linux || darwin +// +build linux darwin + +package command + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/mount" + "github.com/chrislusf/seaweedfs/weed/mount/meta_cache" + "github.com/chrislusf/seaweedfs/weed/mount/unmount" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/security" + "github.com/chrislusf/seaweedfs/weed/storage/types" + "github.com/hanwen/go-fuse/v2/fuse" + "net/http" + "os" + "os/user" + "runtime" + "strconv" + "strings" + "time" + + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/chrislusf/seaweedfs/weed/util/grace" +) + +func runMount2(cmd *Command, args []string) bool { + + if *mountOptions.debug { + go http.ListenAndServe(fmt.Sprintf(":%d", *mountOptions.debugPort), nil) + } + + grace.SetupProfiling(*mountCpuProfile, *mountMemProfile) + if *mountReadRetryTime < time.Second { + *mountReadRetryTime = time.Second + } + util.RetryWaitTime = *mountReadRetryTime + + umask, umaskErr := strconv.ParseUint(*mountOptions.umaskString, 8, 64) + if umaskErr != nil { + fmt.Printf("can not parse umask %s", *mountOptions.umaskString) + return false + } + + if len(args) > 0 { + return false + } + + return RunMount2(&mount2Options, os.FileMode(umask)) +} + +func RunMount2(option *Mount2Options, umask os.FileMode) bool { + + // basic checks + chunkSizeLimitMB := *mountOptions.chunkSizeLimitMB + if chunkSizeLimitMB <= 0 { + fmt.Printf("Please specify a reasonable buffer size.") + return false + } + + // try to connect to filer + filerAddresses := pb.ServerAddresses(*option.filer).ToAddresses() + util.LoadConfiguration("security", false) + grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") + var cipher bool + var err error + for i := 0; i < 10; i++ { + err = pb.WithOneOfGrpcFilerClients(false, filerAddresses, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) + if err != nil { + return fmt.Errorf("get filer grpc address %v configuration: %v", filerAddresses, err) + } + cipher = resp.Cipher + return nil + }) + if err != nil { + glog.V(0).Infof("failed to talk to filer %v: %v", filerAddresses, err) + glog.V(0).Infof("wait for %d seconds ...", i+1) + time.Sleep(time.Duration(i+1) * time.Second) + } + } + if err != nil { + glog.Errorf("failed to talk to filer %v: %v", filerAddresses, err) + return true + } + + filerMountRootPath := *option.filerMountRootPath + + // clean up mount point + dir := util.ResolvePath(*option.dir) + if dir == "" { + fmt.Printf("Please specify the mount directory via \"-dir\"") + return false + } + + unmount.Unmount(dir) + + // detect mount folder mode + if *option.dirAutoCreate { + os.MkdirAll(dir, os.FileMode(0777)&^umask) + } + fileInfo, err := os.Stat(dir) + + // collect uid, gid + uid, gid := uint32(0), uint32(0) + mountMode := os.ModeDir | 0777 + if err == nil { + mountMode = os.ModeDir | os.FileMode(0777)&^umask + uid, gid = util.GetFileUidGid(fileInfo) + fmt.Printf("mount point owner uid=%d gid=%d mode=%s\n", uid, gid, mountMode) + } else { + fmt.Printf("can not stat %s\n", dir) + return false + } + + // detect uid, gid + if uid == 0 { + if u, err := user.Current(); err == nil { + if parsedId, pe := strconv.ParseUint(u.Uid, 10, 32); pe == nil { + uid = uint32(parsedId) + } + if parsedId, pe := strconv.ParseUint(u.Gid, 10, 32); pe == nil { + gid = uint32(parsedId) + } + fmt.Printf("current uid=%d gid=%d\n", uid, gid) + } + } + + // mapping uid, gid + uidGidMapper, err := meta_cache.NewUidGidMapper(*option.uidMap, *option.gidMap) + if err != nil { + fmt.Printf("failed to parse %s %s: %v\n", *option.uidMap, *option.gidMap, err) + return false + } + + // Ensure target mount point availability + if isValid := checkMountPointAvailable(dir); !isValid { + glog.Fatalf("Expected mount to still be active, target mount point: %s, please check!", dir) + return true + } + + // mount fuse + fuseMountOptions := &fuse.MountOptions{ + AllowOther: *option.allowOthers, + Options: nil, + MaxBackground: 128, + MaxWrite: 1024 * 1024 * 2, + MaxReadAhead: 1024 * 1024 * 2, + IgnoreSecurityLabels: false, + RememberInodes: false, + FsName: *option.filer + ":" + filerMountRootPath, + Name: "seaweedfs", + SingleThreaded: false, + DisableXAttrs: false, + Debug: false, + EnableLocks: false, + ExplicitDataCacheControl: false, + // SyncRead: false, // set to false to enable the FUSE_CAP_ASYNC_READ capability + DirectMount: true, + DirectMountFlags: 0, + // EnableAcl: false, + } + + // find mount point + mountRoot := filerMountRootPath + if mountRoot != "/" && strings.HasSuffix(mountRoot, "/") { + mountRoot = mountRoot[0 : len(mountRoot)-1] + } + + seaweedFileSystem := mount.NewSeaweedFileSystem(&mount.Option{ + MountDirectory: dir, + FilerAddresses: filerAddresses, + GrpcDialOption: grpcDialOption, + FilerMountRootPath: mountRoot, + Collection: *option.collection, + Replication: *option.replication, + TtlSec: int32(*option.ttlSec), + DiskType: types.ToDiskType(*option.diskType), + ChunkSizeLimit: int64(chunkSizeLimitMB) * 1024 * 1024, + ConcurrentWriters: *option.concurrentWriters, + CacheDir: *option.cacheDir, + CacheSizeMB: *option.cacheSizeMB, + DataCenter: *option.dataCenter, + MountUid: uid, + MountGid: gid, + MountMode: mountMode, + MountCtime: fileInfo.ModTime(), + MountMtime: time.Now(), + Umask: umask, + VolumeServerAccess: *mountOptions.volumeServerAccess, + Cipher: cipher, + UidGidMapper: uidGidMapper, + }) + + server, err := fuse.NewServer(seaweedFileSystem, dir, fuseMountOptions) + if err != nil { + glog.Fatalf("Mount fail: %v", err) + } + grace.OnInterrupt(func() { + unmount.Unmount(dir) + }) + + seaweedFileSystem.StartBackgroundTasks() + + fmt.Printf("This is SeaweedFS version %s %s %s\n", util.Version(), runtime.GOOS, runtime.GOARCH) + + server.Serve() + + return true +} diff --git a/weed/command/server.go b/weed/command/server.go index 01c59fb85..bab77de15 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -98,6 +98,8 @@ func init() { masterOptions.metricsAddress = cmdServer.Flag.String("metrics.address", "", "Prometheus gateway address") masterOptions.metricsIntervalSec = cmdServer.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds") masterOptions.raftResumeState = cmdServer.Flag.Bool("resumeState", false, "resume previous state on start master server") + masterOptions.heartbeatInterval = cmdServer.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)") + masterOptions.electionTimeout = cmdServer.Flag.Duration("master.electionTimeout", 10*time.Second, "election timeout of master servers") filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection") filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port") @@ -145,7 +147,7 @@ func init() { webdavOptions.tlsPrivateKey = cmdServer.Flag.String("webdav.key.file", "", "path to the TLS private key file") webdavOptions.tlsCertificate = cmdServer.Flag.String("webdav.cert.file", "", "path to the TLS certificate file") webdavOptions.cacheDir = cmdServer.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks") - webdavOptions.cacheSizeMB = cmdServer.Flag.Int64("webdav.cacheCapacityMB", 1000, "local cache capacity in MB") + webdavOptions.cacheSizeMB = cmdServer.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB") msgBrokerOptions.port = cmdServer.Flag.Int("msgBroker.port", 17777, "broker gRPC listen port") diff --git a/weed/command/volume.go b/weed/command/volume.go index 5b9d94b9a..20935bf16 100644 --- a/weed/command/volume.go +++ b/weed/command/volume.go @@ -364,8 +364,8 @@ func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) httpd } httpDown := httpdown.HTTP{ - KillTimeout: 5 * time.Minute, - StopTimeout: 5 * time.Minute, + KillTimeout: time.Minute, + StopTimeout: 30 * time.Second, CertFile: certFile, KeyFile: keyFile} clusterHttpServer := httpDown.Serve(&http.Server{Handler: handler}, listener) diff --git a/weed/command/webdav.go b/weed/command/webdav.go index 319302175..689bf3c30 100644 --- a/weed/command/webdav.go +++ b/weed/command/webdav.go @@ -43,7 +43,7 @@ func init() { webDavStandaloneOptions.tlsPrivateKey = cmdWebDav.Flag.String("key.file", "", "path to the TLS private key file") webDavStandaloneOptions.tlsCertificate = cmdWebDav.Flag.String("cert.file", "", "path to the TLS certificate file") webDavStandaloneOptions.cacheDir = cmdWebDav.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks") - webDavStandaloneOptions.cacheSizeMB = cmdWebDav.Flag.Int64("cacheCapacityMB", 1000, "local cache capacity in MB") + webDavStandaloneOptions.cacheSizeMB = cmdWebDav.Flag.Int64("cacheCapacityMB", 0, "local cache capacity in MB") } var cmdWebDav = &Command{ diff --git a/weed/filer/filechunks.go b/weed/filer/filechunks.go index be18d45ac..d18d06f2c 100644 --- a/weed/filer/filechunks.go +++ b/weed/filer/filechunks.go @@ -101,6 +101,21 @@ func DoMinusChunks(as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk) { return } +func DoMinusChunksBySourceFileId(as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk) { + + fileIds := make(map[string]bool) + for _, interval := range bs { + fileIds[interval.GetFileIdString()] = true + } + for _, chunk := range as { + if _, found := fileIds[chunk.GetSourceFileId()]; !found { + delta = append(delta, chunk) + } + } + + return +} + type ChunkView struct { FileId string Offset int64 diff --git a/weed/filer/filer_conf.go b/weed/filer/filer_conf.go index 45c368b9b..32fc647d9 100644 --- a/weed/filer/filer_conf.go +++ b/weed/filer/filer_conf.go @@ -75,12 +75,12 @@ func (fc *FilerConf) loadFromFiler(filer *Filer) (err error) { return fc.LoadFromBytes(entry.Content) } - return fc.loadFromChunks(filer, entry.Content, entry.Chunks) + return fc.loadFromChunks(filer, entry.Content, entry.Chunks, entry.Size()) } -func (fc *FilerConf) loadFromChunks(filer *Filer, content []byte, chunks []*filer_pb.FileChunk) (err error) { +func (fc *FilerConf) loadFromChunks(filer *Filer, content []byte, chunks []*filer_pb.FileChunk, size uint64) (err error) { if len(content) == 0 { - content, err = filer.readEntry(chunks) + content, err = filer.readEntry(chunks, size) if err != nil { glog.Errorf("read filer conf content: %v", err) return diff --git a/weed/filer/filer_on_meta_event.go b/weed/filer/filer_on_meta_event.go index 34ac5321a..720e019f4 100644 --- a/weed/filer/filer_on_meta_event.go +++ b/weed/filer/filer_on_meta_event.go @@ -2,8 +2,6 @@ package filer import ( "bytes" - "math" - "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" @@ -55,9 +53,9 @@ func (f *Filer) maybeReloadFilerConfiguration(event *filer_pb.SubscribeMetadataR } } -func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) { +func (f *Filer) readEntry(chunks []*filer_pb.FileChunk, size uint64) ([]byte, error) { var buf bytes.Buffer - err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64) + err := StreamContent(f.MasterClient, &buf, chunks, 0, int64(size)) if err != nil { return nil, err } @@ -66,7 +64,7 @@ func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) { func (f *Filer) reloadFilerConfiguration(entry *filer_pb.Entry) { fc := NewFilerConf() - err := fc.loadFromChunks(f, entry.Content, entry.Chunks) + err := fc.loadFromChunks(f, entry.Content, entry.Chunks, FileSize(entry)) if err != nil { glog.Errorf("read filer conf chunks: %v", err) return diff --git a/weed/filer/leveldb/leveldb_store_test.go b/weed/filer/leveldb/leveldb_store_test.go index 2476e063c..c8ec76e23 100644 --- a/weed/filer/leveldb/leveldb_store_test.go +++ b/weed/filer/leveldb/leveldb_store_test.go @@ -13,8 +13,7 @@ import ( func TestCreateAndFind(t *testing.T) { testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil) - dir, _ := os.MkdirTemp("", "seaweedfs_filer_test") - defer os.RemoveAll(dir) + dir := t.TempDir() store := &LevelDBStore{} store.initialize(dir) testFiler.SetStore(store) @@ -67,8 +66,7 @@ func TestCreateAndFind(t *testing.T) { func TestEmptyRoot(t *testing.T) { testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil) - dir, _ := os.MkdirTemp("", "seaweedfs_filer_test2") - defer os.RemoveAll(dir) + dir := t.TempDir() store := &LevelDBStore{} store.initialize(dir) testFiler.SetStore(store) @@ -90,8 +88,7 @@ func TestEmptyRoot(t *testing.T) { func BenchmarkInsertEntry(b *testing.B) { testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil) - dir, _ := os.MkdirTemp("", "seaweedfs_filer_bench") - defer os.RemoveAll(dir) + dir := b.TempDir() store := &LevelDBStore{} store.initialize(dir) testFiler.SetStore(store) diff --git a/weed/filer/leveldb2/leveldb2_store_test.go b/weed/filer/leveldb2/leveldb2_store_test.go index 93c622fd9..c5393dcaf 100644 --- a/weed/filer/leveldb2/leveldb2_store_test.go +++ b/weed/filer/leveldb2/leveldb2_store_test.go @@ -2,7 +2,6 @@ package leveldb import ( "context" - "os" "testing" "github.com/chrislusf/seaweedfs/weed/filer" @@ -11,8 +10,7 @@ import ( func TestCreateAndFind(t *testing.T) { testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil) - dir, _ := os.MkdirTemp("", "seaweedfs_filer_test") - defer os.RemoveAll(dir) + dir := t.TempDir() store := &LevelDB2Store{} store.initialize(dir, 2) testFiler.SetStore(store) @@ -65,8 +63,7 @@ func TestCreateAndFind(t *testing.T) { func TestEmptyRoot(t *testing.T) { testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil) - dir, _ := os.MkdirTemp("", "seaweedfs_filer_test2") - defer os.RemoveAll(dir) + dir := t.TempDir() store := &LevelDB2Store{} store.initialize(dir, 2) testFiler.SetStore(store) diff --git a/weed/filer/leveldb3/leveldb3_store_test.go b/weed/filer/leveldb3/leveldb3_store_test.go index a5e97cf10..c70e83507 100644 --- a/weed/filer/leveldb3/leveldb3_store_test.go +++ b/weed/filer/leveldb3/leveldb3_store_test.go @@ -2,7 +2,6 @@ package leveldb import ( "context" - "os" "testing" "github.com/chrislusf/seaweedfs/weed/filer" @@ -11,8 +10,7 @@ import ( func TestCreateAndFind(t *testing.T) { testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil) - dir, _ := os.MkdirTemp("", "seaweedfs_filer_test") - defer os.RemoveAll(dir) + dir := t.TempDir() store := &LevelDB3Store{} store.initialize(dir) testFiler.SetStore(store) @@ -65,8 +63,7 @@ func TestCreateAndFind(t *testing.T) { func TestEmptyRoot(t *testing.T) { testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil) - dir, _ := os.MkdirTemp("", "seaweedfs_filer_test2") - defer os.RemoveAll(dir) + dir := t.TempDir() store := &LevelDB3Store{} store.initialize(dir) testFiler.SetStore(store) diff --git a/weed/filer/read_write.go b/weed/filer/read_write.go index 14e8cab1e..3b6a69fb6 100644 --- a/weed/filer/read_write.go +++ b/weed/filer/read_write.go @@ -4,7 +4,6 @@ import ( "bytes" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/wdclient" - "math" "time" ) @@ -23,7 +22,7 @@ func ReadEntry(masterClient *wdclient.MasterClient, filerClient filer_pb.Seaweed return err } - return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64) + return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, int64(FileSize(respLookupEntry.Entry))) } diff --git a/weed/filer/rocksdb/rocksdb_store_test.go b/weed/filer/rocksdb/rocksdb_store_test.go index fbf8b3112..bcf93cbf5 100644 --- a/weed/filer/rocksdb/rocksdb_store_test.go +++ b/weed/filer/rocksdb/rocksdb_store_test.go @@ -16,8 +16,7 @@ import ( func TestCreateAndFind(t *testing.T) { testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil) - dir, _ := os.MkdirTemp("", "seaweedfs_filer_test") - defer os.RemoveAll(dir) + dir := t.TempDir() store := &RocksDBStore{} store.initialize(dir) testFiler.SetStore(store) @@ -70,8 +69,7 @@ func TestCreateAndFind(t *testing.T) { func TestEmptyRoot(t *testing.T) { testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil) - dir, _ := os.MkdirTemp("", "seaweedfs_filer_test2") - defer os.RemoveAll(dir) + dir := t.TempDir() store := &RocksDBStore{} store.initialize(dir) testFiler.SetStore(store) @@ -93,8 +91,7 @@ func TestEmptyRoot(t *testing.T) { func BenchmarkInsertEntry(b *testing.B) { testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil) - dir, _ := os.MkdirTemp("", "seaweedfs_filer_bench") - defer os.RemoveAll(dir) + dir := b.TempDir() store := &RocksDBStore{} store.initialize(dir) testFiler.SetStore(store) diff --git a/weed/filesys/file.go b/weed/filesys/file.go index 48a024f20..8028d3912 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -72,8 +72,8 @@ func (file *File) Attr(ctx context.Context, attr *fuse.Attr) (err error) { attr.Mtime = time.Unix(entry.Attributes.Mtime, 0) attr.Gid = entry.Attributes.Gid attr.Uid = entry.Attributes.Uid - attr.Blocks = attr.Size/blockSize + 1 - attr.BlockSize = uint32(file.wfs.option.ChunkSizeLimit) + attr.BlockSize = blockSize + attr.Blocks = (attr.Size + blockSize - 1) / blockSize if entry.HardLinkCounter > 0 { attr.Nlink = uint32(entry.HardLinkCounter) } diff --git a/weed/mount/directory.go b/weed/mount/directory.go new file mode 100644 index 000000000..60fbafc37 --- /dev/null +++ b/weed/mount/directory.go @@ -0,0 +1,42 @@ +package mount + +import ( + "bytes" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/hanwen/go-fuse/v2/fs" + "strings" +) + +type Directory struct { + fs.Inode + + name string + wfs *WFS + entry *filer_pb.Entry + parent *Directory + id uint64 +} + +func (dir *Directory) FullPath() string { + var parts []string + for p := dir; p != nil; p = p.parent { + if strings.HasPrefix(p.name, "/") { + if len(p.name) > 1 { + parts = append(parts, p.name[1:]) + } + } else { + parts = append(parts, p.name) + } + } + + if len(parts) == 0 { + return "/" + } + + var buf bytes.Buffer + for i := len(parts) - 1; i >= 0; i-- { + buf.WriteString("/") + buf.WriteString(parts[i]) + } + return buf.String() +} diff --git a/weed/mount/directory_read.go b/weed/mount/directory_read.go new file mode 100644 index 000000000..6034856f0 --- /dev/null +++ b/weed/mount/directory_read.go @@ -0,0 +1,84 @@ +package mount + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/mount/meta_cache" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/hanwen/go-fuse/v2/fs" + "github.com/hanwen/go-fuse/v2/fuse" + "math" + "os" + "syscall" +) + +var _ = fs.NodeReaddirer(&Directory{}) +var _ = fs.NodeGetattrer(&Directory{}) + +func (dir *Directory) Getattr(ctx context.Context, fh fs.FileHandle, out *fuse.AttrOut) syscall.Errno { + out.Mode = 0755 + return 0 +} + +func (dir *Directory) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) { + + dirPath := util.FullPath(dir.FullPath()) + glog.V(4).Infof("Readdir %s", dirPath) + + sourceChan := make(chan fuse.DirEntry, 64) + + stream := newDirectoryListStream(sourceChan) + + processEachEntryFn := func(entry *filer.Entry, isLast bool) { + sourceChan <- fuse.DirEntry{ + Mode: uint32(entry.Mode), + Name: entry.Name(), + Ino: dirPath.Child(entry.Name()).AsInode(os.ModeDir), + } + } + + if err := meta_cache.EnsureVisited(dir.wfs.metaCache, dir.wfs, dirPath); err != nil { + glog.Errorf("dir ReadDirAll %s: %v", dirPath, err) + return nil, fs.ToErrno(os.ErrInvalid) + } + go func() { + dir.wfs.metaCache.ListDirectoryEntries(context.Background(), dirPath, "", false, int64(math.MaxInt32), func(entry *filer.Entry) bool { + processEachEntryFn(entry, false) + return true + }) + close(sourceChan) + }() + + return stream, fs.OK +} + +var _ = fs.DirStream(&DirectoryListStream{}) + +type DirectoryListStream struct { + next fuse.DirEntry + sourceChan chan fuse.DirEntry + isStarted bool + hasNext bool +} + +func newDirectoryListStream(ch chan fuse.DirEntry) *DirectoryListStream { + return &DirectoryListStream{ + sourceChan: ch, + } +} + +func (i *DirectoryListStream) HasNext() bool { + if !i.isStarted { + i.next, i.hasNext = <-i.sourceChan + i.isStarted = true + } + return i.hasNext +} +func (i *DirectoryListStream) Next() (fuse.DirEntry, syscall.Errno) { + t := i.next + i.next, i.hasNext = <-i.sourceChan + return t, fs.OK +} +func (i *DirectoryListStream) Close() { +} diff --git a/weed/mount/dirty_pages_chunked.go b/weed/mount/dirty_pages_chunked.go new file mode 100644 index 000000000..7a9c0afd6 --- /dev/null +++ b/weed/mount/dirty_pages_chunked.go @@ -0,0 +1,99 @@ +package mount + +import ( + "fmt" + "github.com/chrislusf/seaweedfs/weed/filesys/page_writer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "io" + "sync" + "time" +) + +type ChunkedDirtyPages struct { + fh *FileHandle + writeWaitGroup sync.WaitGroup + lastErr error + collection string + replication string + uploadPipeline *page_writer.UploadPipeline + hasWrites bool +} + +var ( + _ = page_writer.DirtyPages(&ChunkedDirtyPages{}) +) + +func newMemoryChunkPages(fh *FileHandle, chunkSize int64) *ChunkedDirtyPages { + + dirtyPages := &ChunkedDirtyPages{ + fh: fh, + } + + dirtyPages.uploadPipeline = page_writer.NewUploadPipeline(fh.wfs.concurrentWriters, chunkSize, dirtyPages.saveChunkedFileIntevalToStorage, fh.wfs.option.ConcurrentWriters) + + return dirtyPages +} + +func (pages *ChunkedDirtyPages) AddPage(offset int64, data []byte) { + pages.hasWrites = true + + glog.V(4).Infof("%v memory AddPage [%d, %d)", pages.fh.fh, offset, offset+int64(len(data))) + pages.uploadPipeline.SaveDataAt(data, offset) + + return +} + +func (pages *ChunkedDirtyPages) FlushData() error { + if !pages.hasWrites { + return nil + } + pages.uploadPipeline.FlushAll() + if pages.lastErr != nil { + return fmt.Errorf("flush data: %v", pages.lastErr) + } + return nil +} + +func (pages *ChunkedDirtyPages) ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64) { + if !pages.hasWrites { + return + } + return pages.uploadPipeline.MaybeReadDataAt(data, startOffset) +} + +func (pages *ChunkedDirtyPages) GetStorageOptions() (collection, replication string) { + return pages.collection, pages.replication +} + +func (pages *ChunkedDirtyPages) saveChunkedFileIntevalToStorage(reader io.Reader, offset int64, size int64, cleanupFn func()) { + + mtime := time.Now().UnixNano() + defer cleanupFn() + + fileFullPath := pages.fh.FullPath() + fileName := fileFullPath.Name() + chunk, collection, replication, err := pages.fh.wfs.saveDataAsChunk(fileFullPath)(reader, fileName, offset) + if err != nil { + glog.V(0).Infof("%v saveToStorage [%d,%d): %v", fileFullPath, offset, offset+size, err) + pages.lastErr = err + return + } + chunk.Mtime = mtime + pages.collection, pages.replication = collection, replication + pages.fh.addChunks([]*filer_pb.FileChunk{chunk}) + pages.fh.entryViewCache = nil + glog.V(3).Infof("%v saveToStorage %s [%d,%d)", fileFullPath, chunk.FileId, offset, offset+size) + +} + +func (pages ChunkedDirtyPages) Destroy() { + pages.uploadPipeline.Shutdown() +} + +func (pages *ChunkedDirtyPages) LockForRead(startOffset, stopOffset int64) { + pages.uploadPipeline.LockForRead(startOffset, stopOffset) +} +func (pages *ChunkedDirtyPages) UnlockForRead(startOffset, stopOffset int64) { + pages.uploadPipeline.UnlockForRead(startOffset, stopOffset) +} diff --git a/weed/mount/filehandle.go b/weed/mount/filehandle.go new file mode 100644 index 000000000..f2a2ec69c --- /dev/null +++ b/weed/mount/filehandle.go @@ -0,0 +1,95 @@ +package mount + +import ( + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" + "io" + "sort" + "sync" +) + +type FileHandleId uint64 + +type FileHandle struct { + fh FileHandleId + counter int64 + entry *filer_pb.Entry + chunkAddLock sync.Mutex + inode uint64 + wfs *WFS + + // cache file has been written to + dirtyMetadata bool + dirtyPages *PageWriter + entryViewCache []filer.VisibleInterval + reader io.ReaderAt + contentType string + handle uint64 + sync.Mutex + + isDeleted bool +} + +func newFileHandle(wfs *WFS, handleId FileHandleId, inode uint64, entry *filer_pb.Entry) *FileHandle { + fh := &FileHandle{ + fh: handleId, + counter: 1, + inode: inode, + wfs: wfs, + } + // dirtyPages: newContinuousDirtyPages(file, writeOnly), + fh.dirtyPages = newPageWriter(fh, wfs.option.ChunkSizeLimit) + if entry != nil { + entry.Attributes.FileSize = filer.FileSize(entry) + } + + return fh +} + +func (fh *FileHandle) FullPath() util.FullPath { + return fh.wfs.inodeToPath.GetPath(fh.inode) +} + +func (fh *FileHandle) addChunks(chunks []*filer_pb.FileChunk) { + + // find the earliest incoming chunk + newChunks := chunks + earliestChunk := newChunks[0] + for i := 1; i < len(newChunks); i++ { + if lessThan(earliestChunk, newChunks[i]) { + earliestChunk = newChunks[i] + } + } + + if fh.entry == nil { + return + } + + // pick out-of-order chunks from existing chunks + for _, chunk := range fh.entry.Chunks { + if lessThan(earliestChunk, chunk) { + chunks = append(chunks, chunk) + } + } + + // sort incoming chunks + sort.Slice(chunks, func(i, j int) bool { + return lessThan(chunks[i], chunks[j]) + }) + + glog.V(4).Infof("%s existing %d chunks adds %d more", fh.FullPath(), len(fh.entry.Chunks), len(chunks)) + + fh.chunkAddLock.Lock() + fh.entry.Chunks = append(fh.entry.Chunks, newChunks...) + fh.entryViewCache = nil + fh.chunkAddLock.Unlock() +} + +func lessThan(a, b *filer_pb.FileChunk) bool { + if a.Mtime == b.Mtime { + return a.Fid.FileKey < b.Fid.FileKey + } + return a.Mtime < b.Mtime +} diff --git a/weed/mount/filehandle_map.go b/weed/mount/filehandle_map.go new file mode 100644 index 000000000..80cfd02c7 --- /dev/null +++ b/weed/mount/filehandle_map.go @@ -0,0 +1,84 @@ +package mount + +import ( + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "sync" +) + +type FileHandleToInode struct { + sync.RWMutex + nextFh FileHandleId + inode2fh map[uint64]*FileHandle + fh2inode map[FileHandleId]uint64 +} + +func NewFileHandleToInode() *FileHandleToInode { + return &FileHandleToInode{ + inode2fh: make(map[uint64]*FileHandle), + fh2inode: make(map[FileHandleId]uint64), + nextFh: 0, + } +} + +func (i *FileHandleToInode) GetFileHandle(fh FileHandleId) *FileHandle { + i.RLock() + defer i.RUnlock() + inode, found := i.fh2inode[fh] + if found { + return i.inode2fh[inode] + } + return nil +} + +func (i *FileHandleToInode) FindFileHandle(inode uint64) (fh *FileHandle, found bool) { + i.RLock() + defer i.RUnlock() + fh, found = i.inode2fh[inode] + return +} + +func (i *FileHandleToInode) AcquireFileHandle(wfs *WFS, inode uint64, entry *filer_pb.Entry) *FileHandle { + i.Lock() + defer i.Unlock() + fh, found := i.inode2fh[inode] + if !found { + fh = newFileHandle(wfs, i.nextFh, inode, entry) + i.nextFh++ + i.inode2fh[inode] = fh + i.fh2inode[fh.fh] = inode + } else { + fh.counter++ + } + return fh +} + +func (i *FileHandleToInode) ReleaseByInode(inode uint64) { + i.Lock() + defer i.Unlock() + fh, found := i.inode2fh[inode] + if found { + fh.counter-- + if fh.counter <= 0 { + delete(i.inode2fh, inode) + delete(i.fh2inode, fh.fh) + } + } +} +func (i *FileHandleToInode) ReleaseByHandle(fh FileHandleId) { + i.Lock() + defer i.Unlock() + inode, found := i.fh2inode[fh] + if found { + fhHandle, fhFound := i.inode2fh[inode] + if !fhFound { + delete(i.fh2inode, fh) + } else { + fhHandle.counter-- + if fhHandle.counter <= 0 { + delete(i.inode2fh, inode) + delete(i.fh2inode, fhHandle.fh) + } + } + + } +} diff --git a/weed/mount/filehandle_read.go b/weed/mount/filehandle_read.go new file mode 100644 index 000000000..71166169e --- /dev/null +++ b/weed/mount/filehandle_read.go @@ -0,0 +1,114 @@ +package mount + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "io" + "math" +) + +func (fh *FileHandle) lockForRead(startOffset int64, size int) { + fh.dirtyPages.LockForRead(startOffset, startOffset+int64(size)) +} +func (fh *FileHandle) unlockForRead(startOffset int64, size int) { + fh.dirtyPages.UnlockForRead(startOffset, startOffset+int64(size)) +} + +func (fh *FileHandle) readFromDirtyPages(buff []byte, startOffset int64) (maxStop int64) { + maxStop = fh.dirtyPages.ReadDirtyDataAt(buff, startOffset) + return +} + +func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) { + + fileFullPath := fh.FullPath() + + entry := fh.entry + if entry == nil { + return 0, io.EOF + } + + if entry.IsInRemoteOnly() { + glog.V(4).Infof("download remote entry %s", fileFullPath) + newEntry, err := fh.downloadRemoteEntry(entry) + if err != nil { + glog.V(1).Infof("download remote entry %s: %v", fileFullPath, err) + return 0, err + } + entry = newEntry + } + + fileSize := int64(filer.FileSize(entry)) + + if fileSize == 0 { + glog.V(1).Infof("empty fh %v", fileFullPath) + return 0, io.EOF + } + + if offset+int64(len(buff)) <= int64(len(entry.Content)) { + totalRead := copy(buff, entry.Content[offset:]) + glog.V(4).Infof("file handle read cached %s [%d,%d] %d", fileFullPath, offset, offset+int64(totalRead), totalRead) + return int64(totalRead), nil + } + + var chunkResolveErr error + if fh.entryViewCache == nil { + fh.entryViewCache, chunkResolveErr = filer.NonOverlappingVisibleIntervals(fh.wfs.LookupFn(), entry.Chunks, 0, math.MaxInt64) + if chunkResolveErr != nil { + return 0, fmt.Errorf("fail to resolve chunk manifest: %v", chunkResolveErr) + } + fh.reader = nil + } + + reader := fh.reader + if reader == nil { + chunkViews := filer.ViewFromVisibleIntervals(fh.entryViewCache, 0, math.MaxInt64) + glog.V(4).Infof("file handle read %s [%d,%d) from %d views", fileFullPath, offset, offset+int64(len(buff)), len(chunkViews)) + for _, chunkView := range chunkViews { + glog.V(4).Infof(" read %s [%d,%d) from chunk %+v", fileFullPath, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size), chunkView.FileId) + } + reader = filer.NewChunkReaderAtFromClient(fh.wfs.LookupFn(), chunkViews, fh.wfs.chunkCache, fileSize) + } + fh.reader = reader + + totalRead, err := reader.ReadAt(buff, offset) + + if err != nil && err != io.EOF { + glog.Errorf("file handle read %s: %v", fileFullPath, err) + } + + glog.V(4).Infof("file handle read %s [%d,%d] %d : %v", fileFullPath, offset, offset+int64(totalRead), totalRead, err) + + return int64(totalRead), err +} + +func (fh *FileHandle) downloadRemoteEntry(entry *filer_pb.Entry) (*filer_pb.Entry, error) { + + fileFullPath := fh.FullPath() + dir, _ := fileFullPath.DirAndName() + + err := fh.wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + + request := &filer_pb.CacheRemoteObjectToLocalClusterRequest{ + Directory: string(dir), + Name: entry.Name, + } + + glog.V(4).Infof("download entry: %v", request) + resp, err := client.CacheRemoteObjectToLocalCluster(context.Background(), request) + if err != nil { + return fmt.Errorf("CacheRemoteObjectToLocalCluster file %s: %v", fileFullPath, err) + } + + entry = resp.Entry + + fh.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, resp.Entry)) + + return nil + }) + + return entry, err +} diff --git a/weed/mount/inode_to_path.go b/weed/mount/inode_to_path.go new file mode 100644 index 000000000..ffb0cc02f --- /dev/null +++ b/weed/mount/inode_to_path.go @@ -0,0 +1,161 @@ +package mount + +import ( + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/util" + "sync" +) + +type InodeToPath struct { + sync.RWMutex + nextInodeId uint64 + inode2path map[uint64]*InodeEntry + path2inode map[util.FullPath]uint64 +} +type InodeEntry struct { + util.FullPath + nlookup uint64 + isDirectory bool + isChildrenCached bool +} + +func NewInodeToPath() *InodeToPath { + t := &InodeToPath{ + inode2path: make(map[uint64]*InodeEntry), + path2inode: make(map[util.FullPath]uint64), + nextInodeId: 2, // the root inode id is 1 + } + t.inode2path[1] = &InodeEntry{"/", 1, true, false} + t.path2inode["/"] = 1 + return t +} + +func (i *InodeToPath) Lookup(path util.FullPath, isDirectory bool) uint64 { + i.Lock() + defer i.Unlock() + inode, found := i.path2inode[path] + if !found { + inode = i.nextInodeId + i.nextInodeId++ + i.path2inode[path] = inode + i.inode2path[inode] = &InodeEntry{path, 1, isDirectory, false} + } else { + i.inode2path[inode].nlookup++ + } + return inode +} + +func (i *InodeToPath) GetInode(path util.FullPath) uint64 { + if path == "/" { + return 1 + } + i.Lock() + defer i.Unlock() + inode, found := i.path2inode[path] + if !found { + // glog.Fatalf("GetInode unknown inode for %s", path) + // this could be the parent for mount point + } + return inode +} + +func (i *InodeToPath) GetPath(inode uint64) util.FullPath { + i.RLock() + defer i.RUnlock() + path, found := i.inode2path[inode] + if !found { + glog.Fatalf("not found inode %d", inode) + } + return path.FullPath +} + +func (i *InodeToPath) HasPath(path util.FullPath) bool { + i.RLock() + defer i.RUnlock() + _, found := i.path2inode[path] + return found +} + +func (i *InodeToPath) MarkChildrenCached(fullpath util.FullPath) { + i.RLock() + defer i.RUnlock() + inode, found := i.path2inode[fullpath] + if !found { + glog.Fatalf("MarkChildrenCached not found inode %v", fullpath) + } + path, found := i.inode2path[inode] + path.isChildrenCached = true +} + +func (i *InodeToPath) IsChildrenCached(fullpath util.FullPath) bool { + i.RLock() + defer i.RUnlock() + inode, found := i.path2inode[fullpath] + if !found { + return false + } + path, found := i.inode2path[inode] + if found { + return path.isChildrenCached + } + return false +} + +func (i *InodeToPath) HasInode(inode uint64) bool { + if inode == 1 { + return true + } + i.RLock() + defer i.RUnlock() + _, found := i.inode2path[inode] + return found +} + +func (i *InodeToPath) RemovePath(path util.FullPath) { + i.Lock() + defer i.Unlock() + inode, found := i.path2inode[path] + if found { + delete(i.path2inode, path) + delete(i.inode2path, inode) + } +} + +func (i *InodeToPath) MovePath(sourcePath, targetPath util.FullPath) { + i.Lock() + defer i.Unlock() + sourceInode, sourceFound := i.path2inode[sourcePath] + targetInode, targetFound := i.path2inode[targetPath] + if sourceFound { + delete(i.path2inode, sourcePath) + i.path2inode[targetPath] = sourceInode + } else { + // it is possible some source folder items has not been visited before + // so no need to worry about their source inodes + return + } + i.inode2path[sourceInode].FullPath = targetPath + if targetFound { + delete(i.inode2path, targetInode) + } else { + i.inode2path[sourceInode].nlookup++ + } +} + +func (i *InodeToPath) Forget(inode, nlookup uint64, onForgetDir func(dir util.FullPath)) { + i.Lock() + path, found := i.inode2path[inode] + if found { + path.nlookup -= nlookup + if path.nlookup <= 0 { + delete(i.path2inode, path.FullPath) + delete(i.inode2path, inode) + } + } + i.Unlock() + if found { + if path.isDirectory && onForgetDir != nil { + onForgetDir(path.FullPath) + } + } +} diff --git a/weed/mount/meta_cache/cache_config.go b/weed/mount/meta_cache/cache_config.go new file mode 100644 index 000000000..e6593ebde --- /dev/null +++ b/weed/mount/meta_cache/cache_config.go @@ -0,0 +1,32 @@ +package meta_cache + +import "github.com/chrislusf/seaweedfs/weed/util" + +var ( + _ = util.Configuration(&cacheConfig{}) +) + +// implementing util.Configuraion +type cacheConfig struct { + dir string +} + +func (c cacheConfig) GetString(key string) string { + return c.dir +} + +func (c cacheConfig) GetBool(key string) bool { + panic("implement me") +} + +func (c cacheConfig) GetInt(key string) int { + panic("implement me") +} + +func (c cacheConfig) GetStringSlice(key string) []string { + panic("implement me") +} + +func (c cacheConfig) SetDefault(key string, value interface{}) { + panic("implement me") +} diff --git a/weed/mount/meta_cache/id_mapper.go b/weed/mount/meta_cache/id_mapper.go new file mode 100644 index 000000000..4a2179f31 --- /dev/null +++ b/weed/mount/meta_cache/id_mapper.go @@ -0,0 +1,101 @@ +package meta_cache + +import ( + "fmt" + "strconv" + "strings" +) + +type UidGidMapper struct { + uidMapper *IdMapper + gidMapper *IdMapper +} + +type IdMapper struct { + localToFiler map[uint32]uint32 + filerToLocal map[uint32]uint32 +} + +// UidGidMapper translates local uid/gid to filer uid/gid +// The local storage always persists the same as the filer. +// The local->filer translation happens when updating the filer first and later saving to meta_cache. +// And filer->local happens when reading from the meta_cache. +func NewUidGidMapper(uidPairsStr, gidPairStr string) (*UidGidMapper, error) { + uidMapper, err := newIdMapper(uidPairsStr) + if err != nil { + return nil, err + } + gidMapper, err := newIdMapper(gidPairStr) + if err != nil { + return nil, err + } + + return &UidGidMapper{ + uidMapper: uidMapper, + gidMapper: gidMapper, + }, nil +} + +func (m *UidGidMapper) LocalToFiler(uid, gid uint32) (uint32, uint32) { + return m.uidMapper.LocalToFiler(uid), m.gidMapper.LocalToFiler(gid) +} +func (m *UidGidMapper) FilerToLocal(uid, gid uint32) (uint32, uint32) { + return m.uidMapper.FilerToLocal(uid), m.gidMapper.FilerToLocal(gid) +} + +func (m *IdMapper) LocalToFiler(id uint32) uint32 { + value, found := m.localToFiler[id] + if found { + return value + } + return id +} +func (m *IdMapper) FilerToLocal(id uint32) uint32 { + value, found := m.filerToLocal[id] + if found { + return value + } + return id +} + +func newIdMapper(pairsStr string) (*IdMapper, error) { + + localToFiler, filerToLocal, err := parseUint32Pairs(pairsStr) + if err != nil { + return nil, err + } + + return &IdMapper{ + localToFiler: localToFiler, + filerToLocal: filerToLocal, + }, nil + +} + +func parseUint32Pairs(pairsStr string) (localToFiler, filerToLocal map[uint32]uint32, err error) { + + if pairsStr == "" { + return + } + + localToFiler = make(map[uint32]uint32) + filerToLocal = make(map[uint32]uint32) + for _, pairStr := range strings.Split(pairsStr, ",") { + pair := strings.Split(pairStr, ":") + localUidStr, filerUidStr := pair[0], pair[1] + localUid, localUidErr := strconv.Atoi(localUidStr) + if localUidErr != nil { + err = fmt.Errorf("failed to parse local %s: %v", localUidStr, localUidErr) + return + } + filerUid, filerUidErr := strconv.Atoi(filerUidStr) + if filerUidErr != nil { + err = fmt.Errorf("failed to parse remote %s: %v", filerUidStr, filerUidErr) + return + } + localToFiler[uint32(localUid)] = uint32(filerUid) + filerToLocal[uint32(filerUid)] = uint32(localUid) + } + + return +} diff --git a/weed/mount/meta_cache/meta_cache.go b/weed/mount/meta_cache/meta_cache.go new file mode 100644 index 000000000..7f997c5b0 --- /dev/null +++ b/weed/mount/meta_cache/meta_cache.go @@ -0,0 +1,160 @@ +package meta_cache + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/filer/leveldb" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" + "os" +) + +// need to have logic similar to FilerStoreWrapper +// e.g. fill fileId field for chunks + +type MetaCache struct { + localStore filer.VirtualFilerStore + // sync.RWMutex + uidGidMapper *UidGidMapper + markCachedFn func(fullpath util.FullPath) + isCachedFn func(fullpath util.FullPath) bool + invalidateFunc func(fullpath util.FullPath, entry *filer_pb.Entry) +} + +func NewMetaCache(dbFolder string, uidGidMapper *UidGidMapper, markCachedFn func(path util.FullPath), isCachedFn func(path util.FullPath) bool, invalidateFunc func(util.FullPath, *filer_pb.Entry)) *MetaCache { + return &MetaCache{ + localStore: openMetaStore(dbFolder), + markCachedFn: markCachedFn, + isCachedFn: isCachedFn, + uidGidMapper: uidGidMapper, + invalidateFunc: func(fullpath util.FullPath, entry *filer_pb.Entry) { + invalidateFunc(fullpath, entry) + }, + } +} + +func openMetaStore(dbFolder string) filer.VirtualFilerStore { + + os.RemoveAll(dbFolder) + os.MkdirAll(dbFolder, 0755) + + store := &leveldb.LevelDBStore{} + config := &cacheConfig{ + dir: dbFolder, + } + + if err := store.Initialize(config, ""); err != nil { + glog.Fatalf("Failed to initialize metadata cache store for %s: %+v", store.GetName(), err) + } + + return filer.NewFilerStoreWrapper(store) + +} + +func (mc *MetaCache) InsertEntry(ctx context.Context, entry *filer.Entry) error { + //mc.Lock() + //defer mc.Unlock() + return mc.doInsertEntry(ctx, entry) +} + +func (mc *MetaCache) doInsertEntry(ctx context.Context, entry *filer.Entry) error { + return mc.localStore.InsertEntry(ctx, entry) +} + +func (mc *MetaCache) AtomicUpdateEntryFromFiler(ctx context.Context, oldPath util.FullPath, newEntry *filer.Entry) error { + //mc.Lock() + //defer mc.Unlock() + + oldDir, _ := oldPath.DirAndName() + if mc.isCachedFn(util.FullPath(oldDir)) { + if oldPath != "" { + if newEntry != nil && oldPath == newEntry.FullPath { + // skip the unnecessary deletion + // leave the update to the following InsertEntry operation + } else { + glog.V(3).Infof("DeleteEntry %s", oldPath) + if err := mc.localStore.DeleteEntry(ctx, oldPath); err != nil { + return err + } + } + } + } else { + // println("unknown old directory:", oldDir) + } + + if newEntry != nil { + newDir, _ := newEntry.DirAndName() + if mc.isCachedFn(util.FullPath(newDir)) { + glog.V(3).Infof("InsertEntry %s/%s", newDir, newEntry.Name()) + if err := mc.localStore.InsertEntry(ctx, newEntry); err != nil { + return err + } + } + } + return nil +} + +func (mc *MetaCache) UpdateEntry(ctx context.Context, entry *filer.Entry) error { + //mc.Lock() + //defer mc.Unlock() + return mc.localStore.UpdateEntry(ctx, entry) +} + +func (mc *MetaCache) FindEntry(ctx context.Context, fp util.FullPath) (entry *filer.Entry, err error) { + //mc.RLock() + //defer mc.RUnlock() + entry, err = mc.localStore.FindEntry(ctx, fp) + if err != nil { + return nil, err + } + mc.mapIdFromFilerToLocal(entry) + return +} + +func (mc *MetaCache) DeleteEntry(ctx context.Context, fp util.FullPath) (err error) { + //mc.Lock() + //defer mc.Unlock() + return mc.localStore.DeleteEntry(ctx, fp) +} +func (mc *MetaCache) DeleteFolderChildren(ctx context.Context, fp util.FullPath) (err error) { + //mc.Lock() + //defer mc.Unlock() + return mc.localStore.DeleteFolderChildren(ctx, fp) +} + +func (mc *MetaCache) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) error { + //mc.RLock() + //defer mc.RUnlock() + + if !mc.isCachedFn(dirPath) { + // if this request comes after renaming, it should be fine + glog.Warningf("unsynchronized dir: %v", dirPath) + } + + _, err := mc.localStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, func(entry *filer.Entry) bool { + mc.mapIdFromFilerToLocal(entry) + return eachEntryFunc(entry) + }) + if err != nil { + return err + } + return err +} + +func (mc *MetaCache) Shutdown() { + //mc.Lock() + //defer mc.Unlock() + mc.localStore.Shutdown() +} + +func (mc *MetaCache) mapIdFromFilerToLocal(entry *filer.Entry) { + entry.Attr.Uid, entry.Attr.Gid = mc.uidGidMapper.FilerToLocal(entry.Attr.Uid, entry.Attr.Gid) +} + +func (mc *MetaCache) Debug() { + if debuggable, ok := mc.localStore.(filer.Debuggable); ok { + println("start debugging") + debuggable.Debug(os.Stderr) + } +} diff --git a/weed/mount/meta_cache/meta_cache_init.go b/weed/mount/meta_cache/meta_cache_init.go new file mode 100644 index 000000000..cd9c71668 --- /dev/null +++ b/weed/mount/meta_cache/meta_cache_init.go @@ -0,0 +1,67 @@ +package meta_cache + +import ( + "context" + "fmt" + + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" +) + +func EnsureVisited(mc *MetaCache, client filer_pb.FilerClient, dirPath util.FullPath) error { + + for { + + // the directory children are already cached + // so no need for this and upper directories + if mc.isCachedFn(dirPath) { + return nil + } + + if err := doEnsureVisited(mc, client, dirPath); err != nil { + return err + } + + // continue to parent directory + if dirPath != "/" { + parent, _ := dirPath.DirAndName() + dirPath = util.FullPath(parent) + } else { + break + } + } + + return nil + +} + +func doEnsureVisited(mc *MetaCache, client filer_pb.FilerClient, path util.FullPath) error { + + glog.V(4).Infof("ReadDirAllEntries %s ...", path) + + err := util.Retry("ReadDirAllEntries", func() error { + return filer_pb.ReadDirAllEntries(client, path, "", func(pbEntry *filer_pb.Entry, isLast bool) error { + entry := filer.FromPbEntry(string(path), pbEntry) + if IsHiddenSystemEntry(string(path), entry.Name()) { + return nil + } + if err := mc.doInsertEntry(context.Background(), entry); err != nil { + glog.V(0).Infof("read %s: %v", entry.FullPath, err) + return err + } + return nil + }) + }) + + if err != nil { + err = fmt.Errorf("list %s: %v", path, err) + } + mc.markCachedFn(path) + return err +} + +func IsHiddenSystemEntry(dir, name string) bool { + return dir == "/" && (name == "topics" || name == "etc") +} diff --git a/weed/mount/meta_cache/meta_cache_subscribe.go b/weed/mount/meta_cache/meta_cache_subscribe.go new file mode 100644 index 000000000..881fee08f --- /dev/null +++ b/weed/mount/meta_cache/meta_cache_subscribe.go @@ -0,0 +1,68 @@ +package meta_cache + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" +) + +func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.FilerClient, dir string, lastTsNs int64) error { + + processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error { + message := resp.EventNotification + + for _, sig := range message.Signatures { + if sig == selfSignature && selfSignature != 0 { + return nil + } + } + + dir := resp.Directory + var oldPath util.FullPath + var newEntry *filer.Entry + if message.OldEntry != nil { + oldPath = util.NewFullPath(dir, message.OldEntry.Name) + glog.V(4).Infof("deleting %v", oldPath) + } + + if message.NewEntry != nil { + if message.NewParentPath != "" { + dir = message.NewParentPath + } + key := util.NewFullPath(dir, message.NewEntry.Name) + glog.V(4).Infof("creating %v", key) + newEntry = filer.FromPbEntry(dir, message.NewEntry) + } + err := mc.AtomicUpdateEntryFromFiler(context.Background(), oldPath, newEntry) + if err == nil { + if message.OldEntry != nil && message.NewEntry != nil { + oldKey := util.NewFullPath(resp.Directory, message.OldEntry.Name) + mc.invalidateFunc(oldKey, message.OldEntry) + if message.OldEntry.Name != message.NewEntry.Name { + newKey := util.NewFullPath(dir, message.NewEntry.Name) + mc.invalidateFunc(newKey, message.NewEntry) + } + } else if message.OldEntry == nil && message.NewEntry != nil { + // no need to invaalidate + } else if message.OldEntry != nil && message.NewEntry == nil { + oldKey := util.NewFullPath(resp.Directory, message.OldEntry.Name) + mc.invalidateFunc(oldKey, message.OldEntry) + } + } + + return err + + } + + util.RetryForever("followMetaUpdates", func() error { + return pb.WithFilerClientFollowMetadata(client, "mount", selfSignature, dir, &lastTsNs, selfSignature, processEventFn, true) + }, func(err error) bool { + glog.Errorf("follow metadata updates: %v", err) + return true + }) + + return nil +} diff --git a/weed/mount/page_writer.go b/weed/mount/page_writer.go new file mode 100644 index 000000000..8685b3d15 --- /dev/null +++ b/weed/mount/page_writer.go @@ -0,0 +1,95 @@ +package mount + +import ( + "github.com/chrislusf/seaweedfs/weed/filesys/page_writer" + "github.com/chrislusf/seaweedfs/weed/glog" +) + +type PageWriter struct { + fh *FileHandle + collection string + replication string + chunkSize int64 + + randomWriter page_writer.DirtyPages +} + +var ( + _ = page_writer.DirtyPages(&PageWriter{}) +) + +func newPageWriter(fh *FileHandle, chunkSize int64) *PageWriter { + pw := &PageWriter{ + fh: fh, + chunkSize: chunkSize, + randomWriter: newMemoryChunkPages(fh, chunkSize), + // randomWriter: newTempFileDirtyPages(fh.f, chunkSize), + } + return pw +} + +func (pw *PageWriter) AddPage(offset int64, data []byte) { + + glog.V(4).Infof("%v AddPage [%d, %d)", pw.fh.fh, offset, offset+int64(len(data))) + + chunkIndex := offset / pw.chunkSize + for i := chunkIndex; len(data) > 0; i++ { + writeSize := min(int64(len(data)), (i+1)*pw.chunkSize-offset) + pw.addToOneChunk(i, offset, data[:writeSize]) + offset += writeSize + data = data[writeSize:] + } +} + +func (pw *PageWriter) addToOneChunk(chunkIndex, offset int64, data []byte) { + pw.randomWriter.AddPage(offset, data) +} + +func (pw *PageWriter) FlushData() error { + return pw.randomWriter.FlushData() +} + +func (pw *PageWriter) ReadDirtyDataAt(data []byte, offset int64) (maxStop int64) { + glog.V(4).Infof("ReadDirtyDataAt %v [%d, %d)", pw.fh.fh, offset, offset+int64(len(data))) + + chunkIndex := offset / pw.chunkSize + for i := chunkIndex; len(data) > 0; i++ { + readSize := min(int64(len(data)), (i+1)*pw.chunkSize-offset) + + maxStop = pw.randomWriter.ReadDirtyDataAt(data[:readSize], offset) + + offset += readSize + data = data[readSize:] + } + + return +} + +func (pw *PageWriter) GetStorageOptions() (collection, replication string) { + return pw.randomWriter.GetStorageOptions() +} + +func (pw *PageWriter) LockForRead(startOffset, stopOffset int64) { + pw.randomWriter.LockForRead(startOffset, stopOffset) +} + +func (pw *PageWriter) UnlockForRead(startOffset, stopOffset int64) { + pw.randomWriter.UnlockForRead(startOffset, stopOffset) +} + +func (pw *PageWriter) Destroy() { + pw.randomWriter.Destroy() +} + +func max(x, y int64) int64 { + if x > y { + return x + } + return y +} +func min(x, y int64) int64 { + if x < y { + return x + } + return y +} diff --git a/weed/mount/page_writer/chunk_interval_list.go b/weed/mount/page_writer/chunk_interval_list.go new file mode 100644 index 000000000..e6dc5d1f5 --- /dev/null +++ b/weed/mount/page_writer/chunk_interval_list.go @@ -0,0 +1,115 @@ +package page_writer + +import "math" + +// ChunkWrittenInterval mark one written interval within one page chunk +type ChunkWrittenInterval struct { + StartOffset int64 + stopOffset int64 + prev *ChunkWrittenInterval + next *ChunkWrittenInterval +} + +func (interval *ChunkWrittenInterval) Size() int64 { + return interval.stopOffset - interval.StartOffset +} + +func (interval *ChunkWrittenInterval) isComplete(chunkSize int64) bool { + return interval.stopOffset-interval.StartOffset == chunkSize +} + +// ChunkWrittenIntervalList mark written intervals within one page chunk +type ChunkWrittenIntervalList struct { + head *ChunkWrittenInterval + tail *ChunkWrittenInterval +} + +func newChunkWrittenIntervalList() *ChunkWrittenIntervalList { + list := &ChunkWrittenIntervalList{ + head: &ChunkWrittenInterval{ + StartOffset: -1, + stopOffset: -1, + }, + tail: &ChunkWrittenInterval{ + StartOffset: math.MaxInt64, + stopOffset: math.MaxInt64, + }, + } + list.head.next = list.tail + list.tail.prev = list.head + return list +} + +func (list *ChunkWrittenIntervalList) MarkWritten(startOffset, stopOffset int64) { + interval := &ChunkWrittenInterval{ + StartOffset: startOffset, + stopOffset: stopOffset, + } + list.addInterval(interval) +} + +func (list *ChunkWrittenIntervalList) IsComplete(chunkSize int64) bool { + return list.size() == 1 && list.head.next.isComplete(chunkSize) +} +func (list *ChunkWrittenIntervalList) WrittenSize() (writtenByteCount int64) { + for t := list.head; t != nil; t = t.next { + writtenByteCount += t.Size() + } + return +} + +func (list *ChunkWrittenIntervalList) addInterval(interval *ChunkWrittenInterval) { + + p := list.head + for ; p.next != nil && p.next.StartOffset <= interval.StartOffset; p = p.next { + } + q := list.tail + for ; q.prev != nil && q.prev.stopOffset >= interval.stopOffset; q = q.prev { + } + + if interval.StartOffset <= p.stopOffset && q.StartOffset <= interval.stopOffset { + // merge p and q together + p.stopOffset = q.stopOffset + unlinkNodesBetween(p, q.next) + return + } + if interval.StartOffset <= p.stopOffset { + // merge new interval into p + p.stopOffset = interval.stopOffset + unlinkNodesBetween(p, q) + return + } + if q.StartOffset <= interval.stopOffset { + // merge new interval into q + q.StartOffset = interval.StartOffset + unlinkNodesBetween(p, q) + return + } + + // add the new interval between p and q + unlinkNodesBetween(p, q) + p.next = interval + interval.prev = p + q.prev = interval + interval.next = q + +} + +// unlinkNodesBetween remove all nodes after start and before stop, exclusive +func unlinkNodesBetween(start *ChunkWrittenInterval, stop *ChunkWrittenInterval) { + if start.next == stop { + return + } + start.next.prev = nil + start.next = stop + stop.prev.next = nil + stop.prev = start +} + +func (list *ChunkWrittenIntervalList) size() int { + var count int + for t := list.head; t != nil; t = t.next { + count++ + } + return count - 2 +} diff --git a/weed/mount/page_writer/chunk_interval_list_test.go b/weed/mount/page_writer/chunk_interval_list_test.go new file mode 100644 index 000000000..b22f5eb5d --- /dev/null +++ b/weed/mount/page_writer/chunk_interval_list_test.go @@ -0,0 +1,49 @@ +package page_writer + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func Test_PageChunkWrittenIntervalList(t *testing.T) { + list := newChunkWrittenIntervalList() + + assert.Equal(t, 0, list.size(), "empty list") + + list.MarkWritten(0, 5) + assert.Equal(t, 1, list.size(), "one interval") + + list.MarkWritten(0, 5) + assert.Equal(t, 1, list.size(), "duplicated interval2") + + list.MarkWritten(95, 100) + assert.Equal(t, 2, list.size(), "two intervals") + + list.MarkWritten(50, 60) + assert.Equal(t, 3, list.size(), "three intervals") + + list.MarkWritten(50, 55) + assert.Equal(t, 3, list.size(), "three intervals merge") + + list.MarkWritten(40, 50) + assert.Equal(t, 3, list.size(), "three intervals grow forward") + + list.MarkWritten(50, 65) + assert.Equal(t, 3, list.size(), "three intervals grow backward") + + list.MarkWritten(70, 80) + assert.Equal(t, 4, list.size(), "four intervals") + + list.MarkWritten(60, 70) + assert.Equal(t, 3, list.size(), "three intervals merged") + + list.MarkWritten(59, 71) + assert.Equal(t, 3, list.size(), "covered three intervals") + + list.MarkWritten(5, 59) + assert.Equal(t, 2, list.size(), "covered two intervals") + + list.MarkWritten(70, 99) + assert.Equal(t, 1, list.size(), "covered one intervals") + +} diff --git a/weed/mount/page_writer/dirty_pages.go b/weed/mount/page_writer/dirty_pages.go new file mode 100644 index 000000000..25b747fad --- /dev/null +++ b/weed/mount/page_writer/dirty_pages.go @@ -0,0 +1,30 @@ +package page_writer + +type DirtyPages interface { + AddPage(offset int64, data []byte) + FlushData() error + ReadDirtyDataAt(data []byte, startOffset int64) (maxStop int64) + GetStorageOptions() (collection, replication string) + Destroy() + LockForRead(startOffset, stopOffset int64) + UnlockForRead(startOffset, stopOffset int64) +} + +func max(x, y int64) int64 { + if x > y { + return x + } + return y +} +func min(x, y int64) int64 { + if x < y { + return x + } + return y +} +func minInt(x, y int) int { + if x < y { + return x + } + return y +} diff --git a/weed/mount/page_writer/page_chunk.go b/weed/mount/page_writer/page_chunk.go new file mode 100644 index 000000000..4e8f31425 --- /dev/null +++ b/weed/mount/page_writer/page_chunk.go @@ -0,0 +1,16 @@ +package page_writer + +import ( + "io" +) + +type SaveToStorageFunc func(reader io.Reader, offset int64, size int64, cleanupFn func()) + +type PageChunk interface { + FreeResource() + WriteDataAt(src []byte, offset int64) (n int) + ReadDataAt(p []byte, off int64) (maxStop int64) + IsComplete() bool + WrittenSize() int64 + SaveContent(saveFn SaveToStorageFunc) +} diff --git a/weed/mount/page_writer/page_chunk_mem.go b/weed/mount/page_writer/page_chunk_mem.go new file mode 100644 index 000000000..dfd54c19e --- /dev/null +++ b/weed/mount/page_writer/page_chunk_mem.go @@ -0,0 +1,69 @@ +package page_writer + +import ( + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/chrislusf/seaweedfs/weed/util/mem" +) + +var ( + _ = PageChunk(&MemChunk{}) +) + +type MemChunk struct { + buf []byte + usage *ChunkWrittenIntervalList + chunkSize int64 + logicChunkIndex LogicChunkIndex +} + +func NewMemChunk(logicChunkIndex LogicChunkIndex, chunkSize int64) *MemChunk { + return &MemChunk{ + logicChunkIndex: logicChunkIndex, + chunkSize: chunkSize, + buf: mem.Allocate(int(chunkSize)), + usage: newChunkWrittenIntervalList(), + } +} + +func (mc *MemChunk) FreeResource() { + mem.Free(mc.buf) +} + +func (mc *MemChunk) WriteDataAt(src []byte, offset int64) (n int) { + innerOffset := offset % mc.chunkSize + n = copy(mc.buf[innerOffset:], src) + mc.usage.MarkWritten(innerOffset, innerOffset+int64(n)) + return +} + +func (mc *MemChunk) ReadDataAt(p []byte, off int64) (maxStop int64) { + memChunkBaseOffset := int64(mc.logicChunkIndex) * mc.chunkSize + for t := mc.usage.head.next; t != mc.usage.tail; t = t.next { + logicStart := max(off, int64(mc.logicChunkIndex)*mc.chunkSize+t.StartOffset) + logicStop := min(off+int64(len(p)), memChunkBaseOffset+t.stopOffset) + if logicStart < logicStop { + copy(p[logicStart-off:logicStop-off], mc.buf[logicStart-memChunkBaseOffset:logicStop-memChunkBaseOffset]) + maxStop = max(maxStop, logicStop) + } + } + return +} + +func (mc *MemChunk) IsComplete() bool { + return mc.usage.IsComplete(mc.chunkSize) +} + +func (mc *MemChunk) WrittenSize() int64 { + return mc.usage.WrittenSize() +} + +func (mc *MemChunk) SaveContent(saveFn SaveToStorageFunc) { + if saveFn == nil { + return + } + for t := mc.usage.head.next; t != mc.usage.tail; t = t.next { + reader := util.NewBytesReader(mc.buf[t.StartOffset:t.stopOffset]) + saveFn(reader, int64(mc.logicChunkIndex)*mc.chunkSize+t.StartOffset, t.Size(), func() { + }) + } +} diff --git a/weed/mount/page_writer/page_chunk_swapfile.go b/weed/mount/page_writer/page_chunk_swapfile.go new file mode 100644 index 000000000..486557629 --- /dev/null +++ b/weed/mount/page_writer/page_chunk_swapfile.go @@ -0,0 +1,121 @@ +package page_writer + +import ( + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/chrislusf/seaweedfs/weed/util/mem" + "os" +) + +var ( + _ = PageChunk(&SwapFileChunk{}) +) + +type ActualChunkIndex int + +type SwapFile struct { + dir string + file *os.File + logicToActualChunkIndex map[LogicChunkIndex]ActualChunkIndex + chunkSize int64 +} + +type SwapFileChunk struct { + swapfile *SwapFile + usage *ChunkWrittenIntervalList + logicChunkIndex LogicChunkIndex + actualChunkIndex ActualChunkIndex +} + +func NewSwapFile(dir string, chunkSize int64) *SwapFile { + return &SwapFile{ + dir: dir, + file: nil, + logicToActualChunkIndex: make(map[LogicChunkIndex]ActualChunkIndex), + chunkSize: chunkSize, + } +} +func (sf *SwapFile) FreeResource() { + if sf.file != nil { + sf.file.Close() + os.Remove(sf.file.Name()) + } +} + +func (sf *SwapFile) NewTempFileChunk(logicChunkIndex LogicChunkIndex) (tc *SwapFileChunk) { + if sf.file == nil { + var err error + sf.file, err = os.CreateTemp(sf.dir, "") + if err != nil { + glog.Errorf("create swap file: %v", err) + return nil + } + } + actualChunkIndex, found := sf.logicToActualChunkIndex[logicChunkIndex] + if !found { + actualChunkIndex = ActualChunkIndex(len(sf.logicToActualChunkIndex)) + sf.logicToActualChunkIndex[logicChunkIndex] = actualChunkIndex + } + + return &SwapFileChunk{ + swapfile: sf, + usage: newChunkWrittenIntervalList(), + logicChunkIndex: logicChunkIndex, + actualChunkIndex: actualChunkIndex, + } +} + +func (sc *SwapFileChunk) FreeResource() { +} + +func (sc *SwapFileChunk) WriteDataAt(src []byte, offset int64) (n int) { + innerOffset := offset % sc.swapfile.chunkSize + var err error + n, err = sc.swapfile.file.WriteAt(src, int64(sc.actualChunkIndex)*sc.swapfile.chunkSize+innerOffset) + if err == nil { + sc.usage.MarkWritten(innerOffset, innerOffset+int64(n)) + } else { + glog.Errorf("failed to write swap file %s: %v", sc.swapfile.file.Name(), err) + } + return +} + +func (sc *SwapFileChunk) ReadDataAt(p []byte, off int64) (maxStop int64) { + chunkStartOffset := int64(sc.logicChunkIndex) * sc.swapfile.chunkSize + for t := sc.usage.head.next; t != sc.usage.tail; t = t.next { + logicStart := max(off, chunkStartOffset+t.StartOffset) + logicStop := min(off+int64(len(p)), chunkStartOffset+t.stopOffset) + if logicStart < logicStop { + actualStart := logicStart - chunkStartOffset + int64(sc.actualChunkIndex)*sc.swapfile.chunkSize + if _, err := sc.swapfile.file.ReadAt(p[logicStart-off:logicStop-off], actualStart); err != nil { + glog.Errorf("failed to reading swap file %s: %v", sc.swapfile.file.Name(), err) + break + } + maxStop = max(maxStop, logicStop) + } + } + return +} + +func (sc *SwapFileChunk) IsComplete() bool { + return sc.usage.IsComplete(sc.swapfile.chunkSize) +} + +func (sc *SwapFileChunk) WrittenSize() int64 { + return sc.usage.WrittenSize() +} + +func (sc *SwapFileChunk) SaveContent(saveFn SaveToStorageFunc) { + if saveFn == nil { + return + } + for t := sc.usage.head.next; t != sc.usage.tail; t = t.next { + data := mem.Allocate(int(t.Size())) + sc.swapfile.file.ReadAt(data, t.StartOffset+int64(sc.actualChunkIndex)*sc.swapfile.chunkSize) + reader := util.NewBytesReader(data) + saveFn(reader, int64(sc.logicChunkIndex)*sc.swapfile.chunkSize+t.StartOffset, t.Size(), func() { + }) + mem.Free(data) + } + sc.usage = newChunkWrittenIntervalList() +} diff --git a/weed/mount/page_writer/upload_pipeline.go b/weed/mount/page_writer/upload_pipeline.go new file mode 100644 index 000000000..53641e66d --- /dev/null +++ b/weed/mount/page_writer/upload_pipeline.go @@ -0,0 +1,182 @@ +package page_writer + +import ( + "fmt" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/util" + "sync" + "sync/atomic" + "time" +) + +type LogicChunkIndex int + +type UploadPipeline struct { + filepath util.FullPath + ChunkSize int64 + writableChunks map[LogicChunkIndex]PageChunk + writableChunksLock sync.Mutex + sealedChunks map[LogicChunkIndex]*SealedChunk + sealedChunksLock sync.Mutex + uploaders *util.LimitedConcurrentExecutor + uploaderCount int32 + uploaderCountCond *sync.Cond + saveToStorageFn SaveToStorageFunc + activeReadChunks map[LogicChunkIndex]int + activeReadChunksLock sync.Mutex + bufferChunkLimit int +} + +type SealedChunk struct { + chunk PageChunk + referenceCounter int // track uploading or reading processes +} + +func (sc *SealedChunk) FreeReference(messageOnFree string) { + sc.referenceCounter-- + if sc.referenceCounter == 0 { + glog.V(4).Infof("Free sealed chunk: %s", messageOnFree) + sc.chunk.FreeResource() + } +} + +func NewUploadPipeline(writers *util.LimitedConcurrentExecutor, chunkSize int64, saveToStorageFn SaveToStorageFunc, bufferChunkLimit int) *UploadPipeline { + return &UploadPipeline{ + ChunkSize: chunkSize, + writableChunks: make(map[LogicChunkIndex]PageChunk), + sealedChunks: make(map[LogicChunkIndex]*SealedChunk), + uploaders: writers, + uploaderCountCond: sync.NewCond(&sync.Mutex{}), + saveToStorageFn: saveToStorageFn, + activeReadChunks: make(map[LogicChunkIndex]int), + bufferChunkLimit: bufferChunkLimit, + } +} + +func (up *UploadPipeline) SaveDataAt(p []byte, off int64) (n int) { + up.writableChunksLock.Lock() + defer up.writableChunksLock.Unlock() + + logicChunkIndex := LogicChunkIndex(off / up.ChunkSize) + + memChunk, found := up.writableChunks[logicChunkIndex] + if !found { + if len(up.writableChunks) < up.bufferChunkLimit { + memChunk = NewMemChunk(logicChunkIndex, up.ChunkSize) + } else { + fullestChunkIndex, fullness := LogicChunkIndex(-1), int64(0) + for lci, mc := range up.writableChunks { + chunkFullness := mc.WrittenSize() + if fullness < chunkFullness { + fullestChunkIndex = lci + fullness = chunkFullness + } + } + up.moveToSealed(up.writableChunks[fullestChunkIndex], fullestChunkIndex) + delete(up.writableChunks, fullestChunkIndex) + fmt.Printf("flush chunk %d with %d bytes written", logicChunkIndex, fullness) + memChunk = NewMemChunk(logicChunkIndex, up.ChunkSize) + } + up.writableChunks[logicChunkIndex] = memChunk + } + n = memChunk.WriteDataAt(p, off) + up.maybeMoveToSealed(memChunk, logicChunkIndex) + + return +} + +func (up *UploadPipeline) MaybeReadDataAt(p []byte, off int64) (maxStop int64) { + logicChunkIndex := LogicChunkIndex(off / up.ChunkSize) + + // read from sealed chunks first + up.sealedChunksLock.Lock() + sealedChunk, found := up.sealedChunks[logicChunkIndex] + if found { + sealedChunk.referenceCounter++ + } + up.sealedChunksLock.Unlock() + if found { + maxStop = sealedChunk.chunk.ReadDataAt(p, off) + glog.V(4).Infof("%s read sealed memchunk [%d,%d)", up.filepath, off, maxStop) + sealedChunk.FreeReference(fmt.Sprintf("%s finish reading chunk %d", up.filepath, logicChunkIndex)) + } + + // read from writable chunks last + up.writableChunksLock.Lock() + defer up.writableChunksLock.Unlock() + writableChunk, found := up.writableChunks[logicChunkIndex] + if !found { + return + } + writableMaxStop := writableChunk.ReadDataAt(p, off) + glog.V(4).Infof("%s read writable memchunk [%d,%d)", up.filepath, off, writableMaxStop) + maxStop = max(maxStop, writableMaxStop) + + return +} + +func (up *UploadPipeline) FlushAll() { + up.writableChunksLock.Lock() + defer up.writableChunksLock.Unlock() + + for logicChunkIndex, memChunk := range up.writableChunks { + up.moveToSealed(memChunk, logicChunkIndex) + } + + up.waitForCurrentWritersToComplete() +} + +func (up *UploadPipeline) maybeMoveToSealed(memChunk PageChunk, logicChunkIndex LogicChunkIndex) { + if memChunk.IsComplete() { + up.moveToSealed(memChunk, logicChunkIndex) + } +} + +func (up *UploadPipeline) moveToSealed(memChunk PageChunk, logicChunkIndex LogicChunkIndex) { + atomic.AddInt32(&up.uploaderCount, 1) + glog.V(4).Infof("%s uploaderCount %d ++> %d", up.filepath, up.uploaderCount-1, up.uploaderCount) + + up.sealedChunksLock.Lock() + + if oldMemChunk, found := up.sealedChunks[logicChunkIndex]; found { + oldMemChunk.FreeReference(fmt.Sprintf("%s replace chunk %d", up.filepath, logicChunkIndex)) + } + sealedChunk := &SealedChunk{ + chunk: memChunk, + referenceCounter: 1, // default 1 is for uploading process + } + up.sealedChunks[logicChunkIndex] = sealedChunk + delete(up.writableChunks, logicChunkIndex) + + up.sealedChunksLock.Unlock() + + up.uploaders.Execute(func() { + // first add to the file chunks + sealedChunk.chunk.SaveContent(up.saveToStorageFn) + + // notify waiting process + atomic.AddInt32(&up.uploaderCount, -1) + glog.V(4).Infof("%s uploaderCount %d --> %d", up.filepath, up.uploaderCount+1, up.uploaderCount) + // Lock and Unlock are not required, + // but it may signal multiple times during one wakeup, + // and the waiting goroutine may miss some of them! + up.uploaderCountCond.L.Lock() + up.uploaderCountCond.Broadcast() + up.uploaderCountCond.L.Unlock() + + // wait for readers + for up.IsLocked(logicChunkIndex) { + time.Sleep(59 * time.Millisecond) + } + + // then remove from sealed chunks + up.sealedChunksLock.Lock() + defer up.sealedChunksLock.Unlock() + delete(up.sealedChunks, logicChunkIndex) + sealedChunk.FreeReference(fmt.Sprintf("%s finished uploading chunk %d", up.filepath, logicChunkIndex)) + + }) +} + +func (up *UploadPipeline) Shutdown() { +} diff --git a/weed/mount/page_writer/upload_pipeline_lock.go b/weed/mount/page_writer/upload_pipeline_lock.go new file mode 100644 index 000000000..47a40ba37 --- /dev/null +++ b/weed/mount/page_writer/upload_pipeline_lock.go @@ -0,0 +1,63 @@ +package page_writer + +import ( + "sync/atomic" +) + +func (up *UploadPipeline) LockForRead(startOffset, stopOffset int64) { + startLogicChunkIndex := LogicChunkIndex(startOffset / up.ChunkSize) + stopLogicChunkIndex := LogicChunkIndex(stopOffset / up.ChunkSize) + if stopOffset%up.ChunkSize > 0 { + stopLogicChunkIndex += 1 + } + up.activeReadChunksLock.Lock() + defer up.activeReadChunksLock.Unlock() + for i := startLogicChunkIndex; i < stopLogicChunkIndex; i++ { + if count, found := up.activeReadChunks[i]; found { + up.activeReadChunks[i] = count + 1 + } else { + up.activeReadChunks[i] = 1 + } + } +} + +func (up *UploadPipeline) UnlockForRead(startOffset, stopOffset int64) { + startLogicChunkIndex := LogicChunkIndex(startOffset / up.ChunkSize) + stopLogicChunkIndex := LogicChunkIndex(stopOffset / up.ChunkSize) + if stopOffset%up.ChunkSize > 0 { + stopLogicChunkIndex += 1 + } + up.activeReadChunksLock.Lock() + defer up.activeReadChunksLock.Unlock() + for i := startLogicChunkIndex; i < stopLogicChunkIndex; i++ { + if count, found := up.activeReadChunks[i]; found { + if count == 1 { + delete(up.activeReadChunks, i) + } else { + up.activeReadChunks[i] = count - 1 + } + } + } +} + +func (up *UploadPipeline) IsLocked(logicChunkIndex LogicChunkIndex) bool { + up.activeReadChunksLock.Lock() + defer up.activeReadChunksLock.Unlock() + if count, found := up.activeReadChunks[logicChunkIndex]; found { + return count > 0 + } + return false +} + +func (up *UploadPipeline) waitForCurrentWritersToComplete() { + up.uploaderCountCond.L.Lock() + t := int32(100) + for { + t = atomic.LoadInt32(&up.uploaderCount) + if t <= 0 { + break + } + up.uploaderCountCond.Wait() + } + up.uploaderCountCond.L.Unlock() +} diff --git a/weed/mount/page_writer/upload_pipeline_test.go b/weed/mount/page_writer/upload_pipeline_test.go new file mode 100644 index 000000000..816fb228b --- /dev/null +++ b/weed/mount/page_writer/upload_pipeline_test.go @@ -0,0 +1,47 @@ +package page_writer + +import ( + "github.com/chrislusf/seaweedfs/weed/util" + "testing" +) + +func TestUploadPipeline(t *testing.T) { + + uploadPipeline := NewUploadPipeline(nil, 2*1024*1024, nil, 16) + + writeRange(uploadPipeline, 0, 131072) + writeRange(uploadPipeline, 131072, 262144) + writeRange(uploadPipeline, 262144, 1025536) + + confirmRange(t, uploadPipeline, 0, 1025536) + + writeRange(uploadPipeline, 1025536, 1296896) + + confirmRange(t, uploadPipeline, 1025536, 1296896) + + writeRange(uploadPipeline, 1296896, 2162688) + + confirmRange(t, uploadPipeline, 1296896, 2162688) + + confirmRange(t, uploadPipeline, 1296896, 2162688) +} + +// startOff and stopOff must be divided by 4 +func writeRange(uploadPipeline *UploadPipeline, startOff, stopOff int64) { + p := make([]byte, 4) + for i := startOff / 4; i < stopOff/4; i += 4 { + util.Uint32toBytes(p, uint32(i)) + uploadPipeline.SaveDataAt(p, i) + } +} + +func confirmRange(t *testing.T, uploadPipeline *UploadPipeline, startOff, stopOff int64) { + p := make([]byte, 4) + for i := startOff; i < stopOff/4; i += 4 { + uploadPipeline.MaybeReadDataAt(p, i) + x := util.BytesToUint32(p) + if x != uint32(i) { + t.Errorf("expecting %d found %d at offset [%d,%d)", i, x, i, i+4) + } + } +} diff --git a/weed/mount/unmount/unmount.go b/weed/mount/unmount/unmount.go new file mode 100644 index 000000000..c481d8030 --- /dev/null +++ b/weed/mount/unmount/unmount.go @@ -0,0 +1,6 @@ +package unmount + +// Unmount tries to unmount the filesystem mounted at dir. +func Unmount(dir string) error { + return unmount(dir) +} diff --git a/weed/mount/unmount/unmount_linux.go b/weed/mount/unmount/unmount_linux.go new file mode 100644 index 000000000..e55d48f86 --- /dev/null +++ b/weed/mount/unmount/unmount_linux.go @@ -0,0 +1,21 @@ +package unmount + +import ( + "bytes" + "errors" + "os/exec" +) + +func unmount(dir string) error { + cmd := exec.Command("fusermount", "-u", dir) + output, err := cmd.CombinedOutput() + if err != nil { + if len(output) > 0 { + output = bytes.TrimRight(output, "\n") + msg := err.Error() + ": " + string(output) + err = errors.New(msg) + } + return err + } + return nil +} diff --git a/weed/mount/unmount/unmount_std.go b/weed/mount/unmount/unmount_std.go new file mode 100644 index 000000000..410eb1235 --- /dev/null +++ b/weed/mount/unmount/unmount_std.go @@ -0,0 +1,18 @@ +//go:build !linux && !windows +// +build !linux,!windows + +package unmount + +import ( + "os" + "syscall" +) + +func unmount(dir string) error { + err := syscall.Unmount(dir, 0) + if err != nil { + err = &os.PathError{Op: "unmount", Path: dir, Err: err} + return err + } + return nil +} diff --git a/weed/mount/unmount/unmount_unsupported.go b/weed/mount/unmount/unmount_unsupported.go new file mode 100644 index 000000000..d0a94cc4a --- /dev/null +++ b/weed/mount/unmount/unmount_unsupported.go @@ -0,0 +1,8 @@ +//go:build windows +// +build windows + +package unmount + +func unmount(dir string) error { + return nil +} diff --git a/weed/mount/weedfs.go b/weed/mount/weedfs.go new file mode 100644 index 000000000..072f562fc --- /dev/null +++ b/weed/mount/weedfs.go @@ -0,0 +1,187 @@ +package mount + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/mount/meta_cache" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/storage/types" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/chrislusf/seaweedfs/weed/util/chunk_cache" + "github.com/chrislusf/seaweedfs/weed/util/grace" + "github.com/chrislusf/seaweedfs/weed/wdclient" + "github.com/hanwen/go-fuse/v2/fuse" + "google.golang.org/grpc" + "math/rand" + "os" + "path" + "path/filepath" + "time" + + "github.com/hanwen/go-fuse/v2/fs" +) + +type Option struct { + MountDirectory string + FilerAddresses []pb.ServerAddress + filerIndex int + GrpcDialOption grpc.DialOption + FilerMountRootPath string + Collection string + Replication string + TtlSec int32 + DiskType types.DiskType + ChunkSizeLimit int64 + ConcurrentWriters int + CacheDir string + CacheSizeMB int64 + DataCenter string + Umask os.FileMode + + MountUid uint32 + MountGid uint32 + MountMode os.FileMode + MountCtime time.Time + MountMtime time.Time + MountParentInode uint64 + + VolumeServerAccess string // how to access volume servers + Cipher bool // whether encrypt data on volume server + UidGidMapper *meta_cache.UidGidMapper + + uniqueCacheDir string + uniqueCacheTempPageDir string +} + +type WFS struct { + // follow https://github.com/hanwen/go-fuse/blob/master/fuse/api.go + fuse.RawFileSystem + fs.Inode + option *Option + metaCache *meta_cache.MetaCache + stats statsCache + root Directory + chunkCache *chunk_cache.TieredChunkCache + signature int32 + concurrentWriters *util.LimitedConcurrentExecutor + inodeToPath *InodeToPath + fhmap *FileHandleToInode +} + +func NewSeaweedFileSystem(option *Option) *WFS { + wfs := &WFS{ + RawFileSystem: fuse.NewDefaultRawFileSystem(), + option: option, + signature: util.RandomInt32(), + inodeToPath: NewInodeToPath(), + fhmap: NewFileHandleToInode(), + } + + wfs.root = Directory{ + name: "/", + wfs: wfs, + entry: nil, + parent: nil, + } + + wfs.option.filerIndex = rand.Intn(len(option.FilerAddresses)) + wfs.option.setupUniqueCacheDirectory() + if option.CacheSizeMB > 0 { + wfs.chunkCache = chunk_cache.NewTieredChunkCache(256, option.getUniqueCacheDir(), option.CacheSizeMB, 1024*1024) + } + + wfs.metaCache = meta_cache.NewMetaCache(path.Join(option.getUniqueCacheDir(), "meta"), option.UidGidMapper, func(path util.FullPath) { + wfs.inodeToPath.MarkChildrenCached(path) + }, func(path util.FullPath) bool { + return wfs.inodeToPath.IsChildrenCached(path) + }, func(filePath util.FullPath, entry *filer_pb.Entry) { + }) + grace.OnInterrupt(func() { + wfs.metaCache.Shutdown() + }) + + if wfs.option.ConcurrentWriters > 0 { + wfs.concurrentWriters = util.NewLimitedConcurrentExecutor(wfs.option.ConcurrentWriters) + } + return wfs +} + +func (wfs *WFS) StartBackgroundTasks() { + startTime := time.Now() + go meta_cache.SubscribeMetaEvents(wfs.metaCache, wfs.signature, wfs, wfs.option.FilerMountRootPath, startTime.UnixNano()) +} + +func (wfs *WFS) Root() *Directory { + return &wfs.root +} + +func (wfs *WFS) String() string { + return "seaweedfs" +} + +func (wfs *WFS) maybeReadEntry(inode uint64) (path util.FullPath, fh *FileHandle, entry *filer_pb.Entry, status fuse.Status) { + path = wfs.inodeToPath.GetPath(inode) + var found bool + if fh, found = wfs.fhmap.FindFileHandle(inode); found { + return path, fh, fh.entry, fuse.OK + } + entry, status = wfs.maybeLoadEntry(path) + return +} + +func (wfs *WFS) maybeLoadEntry(fullpath util.FullPath) (*filer_pb.Entry, fuse.Status) { + + // glog.V(3).Infof("read entry cache miss %s", fullpath) + dir, name := fullpath.DirAndName() + + // return a valid entry for the mount root + if string(fullpath) == wfs.option.FilerMountRootPath { + return &filer_pb.Entry{ + Name: name, + IsDirectory: true, + Attributes: &filer_pb.FuseAttributes{ + Mtime: wfs.option.MountMtime.Unix(), + FileMode: uint32(wfs.option.MountMode), + Uid: wfs.option.MountUid, + Gid: wfs.option.MountGid, + Crtime: wfs.option.MountCtime.Unix(), + }, + }, fuse.OK + } + + // read from async meta cache + meta_cache.EnsureVisited(wfs.metaCache, wfs, util.FullPath(dir)) + cachedEntry, cacheErr := wfs.metaCache.FindEntry(context.Background(), fullpath) + if cacheErr == filer_pb.ErrNotFound { + return nil, fuse.ENOENT + } + return cachedEntry.ToProtoEntry(), fuse.OK +} + +func (wfs *WFS) LookupFn() wdclient.LookupFileIdFunctionType { + if wfs.option.VolumeServerAccess == "filerProxy" { + return func(fileId string) (targetUrls []string, err error) { + return []string{"http://" + wfs.getCurrentFiler().ToHttpAddress() + "/?proxyChunkId=" + fileId}, nil + } + } + return filer.LookupFn(wfs) +} + +func (wfs *WFS) getCurrentFiler() pb.ServerAddress { + return wfs.option.FilerAddresses[wfs.option.filerIndex] +} + +func (option *Option) setupUniqueCacheDirectory() { + cacheUniqueId := util.Md5String([]byte(option.MountDirectory + string(option.FilerAddresses[0]) + option.FilerMountRootPath + util.Version()))[0:8] + option.uniqueCacheDir = path.Join(option.CacheDir, cacheUniqueId) + option.uniqueCacheTempPageDir = filepath.Join(option.uniqueCacheDir, "sw") + os.MkdirAll(option.uniqueCacheTempPageDir, os.FileMode(0777)&^option.Umask) +} + +func (option *Option) getTempFilePageDir() string { + return option.uniqueCacheTempPageDir +} +func (option *Option) getUniqueCacheDir() string { + return option.uniqueCacheDir +} diff --git a/weed/mount/weedfs_attr.go b/weed/mount/weedfs_attr.go new file mode 100644 index 000000000..4042ce8f1 --- /dev/null +++ b/weed/mount/weedfs_attr.go @@ -0,0 +1,207 @@ +package mount + +import ( + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/hanwen/go-fuse/v2/fuse" + "os" + "syscall" + "time" +) + +func (wfs *WFS) GetAttr(cancel <-chan struct{}, input *fuse.GetAttrIn, out *fuse.AttrOut) (code fuse.Status) { + if input.NodeId == 1 { + wfs.setRootAttr(out) + return fuse.OK + } + + _, _, entry, status := wfs.maybeReadEntry(input.NodeId) + if status != fuse.OK { + return status + } + out.AttrValid = 1 + wfs.setAttrByPbEntry(&out.Attr, input.NodeId, entry) + + return fuse.OK +} + +func (wfs *WFS) SetAttr(cancel <-chan struct{}, input *fuse.SetAttrIn, out *fuse.AttrOut) (code fuse.Status) { + + path, fh, entry, status := wfs.maybeReadEntry(input.NodeId) + if status != fuse.OK { + return status + } + + if size, ok := input.GetSize(); ok { + glog.V(4).Infof("%v setattr set size=%v chunks=%d", path, size, len(entry.Chunks)) + if size < filer.FileSize(entry) { + // fmt.Printf("truncate %v \n", fullPath) + var chunks []*filer_pb.FileChunk + var truncatedChunks []*filer_pb.FileChunk + for _, chunk := range entry.Chunks { + int64Size := int64(chunk.Size) + if chunk.Offset+int64Size > int64(size) { + // this chunk is truncated + int64Size = int64(size) - chunk.Offset + if int64Size > 0 { + chunks = append(chunks, chunk) + glog.V(4).Infof("truncated chunk %+v from %d to %d\n", chunk.GetFileIdString(), chunk.Size, int64Size) + chunk.Size = uint64(int64Size) + } else { + glog.V(4).Infof("truncated whole chunk %+v\n", chunk.GetFileIdString()) + truncatedChunks = append(truncatedChunks, chunk) + } + } + } + // set the new chunks and reset entry cache + entry.Chunks = chunks + if fh != nil { + fh.entryViewCache = nil + } + } + entry.Attributes.Mtime = time.Now().Unix() + entry.Attributes.FileSize = size + + } + + if mode, ok := input.GetMode(); ok { + entry.Attributes.FileMode = uint32(mode) + } + + if uid, ok := input.GetUID(); ok { + entry.Attributes.Uid = uid + } + + if gid, ok := input.GetGID(); ok { + entry.Attributes.Gid = gid + } + + if mtime, ok := input.GetMTime(); ok { + entry.Attributes.Mtime = mtime.Unix() + } + + entry.Attributes.Mtime = time.Now().Unix() + out.AttrValid = 1 + wfs.setAttrByPbEntry(&out.Attr, input.NodeId, entry) + + if fh != nil { + fh.dirtyMetadata = true + return fuse.OK + } + + return wfs.saveEntry(path, entry) + +} + +func (wfs *WFS) setRootAttr(out *fuse.AttrOut) { + now := uint64(time.Now().Unix()) + out.AttrValid = 119 + out.Ino = 1 + setBlksize(&out.Attr, blockSize) + out.Uid = wfs.option.MountUid + out.Gid = wfs.option.MountGid + out.Mtime = now + out.Ctime = now + out.Atime = now + out.Mode = toSystemType(os.ModeDir) | uint32(wfs.option.MountMode) + out.Nlink = 1 +} + +func (wfs *WFS) setAttrByPbEntry(out *fuse.Attr, inode uint64, entry *filer_pb.Entry) { + out.Ino = inode + out.Size = filer.FileSize(entry) + out.Blocks = (out.Size + blockSize - 1) / blockSize + setBlksize(out, blockSize) + out.Mtime = uint64(entry.Attributes.Mtime) + out.Ctime = uint64(entry.Attributes.Mtime) + out.Atime = uint64(entry.Attributes.Mtime) + out.Mode = toSystemMode(os.FileMode(entry.Attributes.FileMode)) + if entry.HardLinkCounter > 0 { + out.Nlink = uint32(entry.HardLinkCounter) + } else { + out.Nlink = 1 + } + out.Uid = entry.Attributes.Uid + out.Gid = entry.Attributes.Gid +} + +func (wfs *WFS) setAttrByFilerEntry(out *fuse.Attr, inode uint64, entry *filer.Entry) { + out.Ino = inode + out.Size = entry.FileSize + out.Blocks = (out.Size + blockSize - 1) / blockSize + setBlksize(out, blockSize) + out.Atime = uint64(entry.Attr.Mtime.Unix()) + out.Mtime = uint64(entry.Attr.Mtime.Unix()) + out.Ctime = uint64(entry.Attr.Mtime.Unix()) + out.Mode = toSystemMode(entry.Attr.Mode) + if entry.HardLinkCounter > 0 { + out.Nlink = uint32(entry.HardLinkCounter) + } else { + out.Nlink = 1 + } + out.Uid = entry.Attr.Uid + out.Gid = entry.Attr.Gid +} + +func (wfs *WFS) outputPbEntry(out *fuse.EntryOut, inode uint64, entry *filer_pb.Entry) { + out.NodeId = inode + out.Generation = 1 + out.EntryValid = 1 + out.AttrValid = 1 + wfs.setAttrByPbEntry(&out.Attr, inode, entry) +} + +func (wfs *WFS) outputFilerEntry(out *fuse.EntryOut, inode uint64, entry *filer.Entry) { + out.NodeId = inode + out.Generation = 1 + out.EntryValid = 1 + out.AttrValid = 1 + wfs.setAttrByFilerEntry(&out.Attr, inode, entry) +} + +func toSystemMode(mode os.FileMode) uint32 { + return toSystemType(mode) | uint32(mode) +} + +func toSystemType(mode os.FileMode) uint32 { + switch mode & os.ModeType { + case os.ModeDir: + return syscall.S_IFDIR + case os.ModeSymlink: + return syscall.S_IFLNK + case os.ModeNamedPipe: + return syscall.S_IFIFO + case os.ModeSocket: + return syscall.S_IFSOCK + case os.ModeDevice: + return syscall.S_IFBLK + case os.ModeCharDevice: + return syscall.S_IFCHR + default: + return syscall.S_IFREG + } +} + +func toFileType(mode uint32) os.FileMode { + switch mode & (syscall.S_IFMT & 0xffff) { + case syscall.S_IFDIR: + return os.ModeDir + case syscall.S_IFLNK: + return os.ModeSymlink + case syscall.S_IFIFO: + return os.ModeNamedPipe + case syscall.S_IFSOCK: + return os.ModeSocket + case syscall.S_IFBLK: + return os.ModeDevice + case syscall.S_IFCHR: + return os.ModeCharDevice + default: + return 0 + } +} + +func toFileMode(mode uint32) os.FileMode { + return toFileType(mode) | os.FileMode(mode&07777) +} diff --git a/weed/mount/weedfs_attr_darwin.go b/weed/mount/weedfs_attr_darwin.go new file mode 100644 index 000000000..e7767d4a6 --- /dev/null +++ b/weed/mount/weedfs_attr_darwin.go @@ -0,0 +1,8 @@ +package mount + +import ( + "github.com/hanwen/go-fuse/v2/fuse" +) + +func setBlksize(out *fuse.Attr, size uint32) { +} diff --git a/weed/mount/weedfs_attr_linux.go b/weed/mount/weedfs_attr_linux.go new file mode 100644 index 000000000..56be62e62 --- /dev/null +++ b/weed/mount/weedfs_attr_linux.go @@ -0,0 +1,9 @@ +package mount + +import ( + "github.com/hanwen/go-fuse/v2/fuse" +) + +func setBlksize(out *fuse.Attr, size uint32) { + out.Blksize = size +} diff --git a/weed/mount/weedfs_dir_lookup.go b/weed/mount/weedfs_dir_lookup.go new file mode 100644 index 000000000..30b61d75f --- /dev/null +++ b/weed/mount/weedfs_dir_lookup.go @@ -0,0 +1,59 @@ +package mount + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/mount/meta_cache" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/hanwen/go-fuse/v2/fuse" +) + +// Lookup is called by the kernel when the VFS wants to know +// about a file inside a directory. Many lookup calls can +// occur in parallel, but only one call happens for each (dir, +// name) pair. + +func (wfs *WFS) Lookup(cancel <-chan struct{}, header *fuse.InHeader, name string, out *fuse.EntryOut) (code fuse.Status) { + + if s := checkName(name); s != fuse.OK { + return s + } + + dirPath := wfs.inodeToPath.GetPath(header.NodeId) + + fullFilePath := dirPath.Child(name) + + visitErr := meta_cache.EnsureVisited(wfs.metaCache, wfs, dirPath) + if visitErr != nil { + glog.Errorf("dir Lookup %s: %v", dirPath, visitErr) + return fuse.EIO + } + localEntry, cacheErr := wfs.metaCache.FindEntry(context.Background(), fullFilePath) + if cacheErr == filer_pb.ErrNotFound { + return fuse.ENOENT + } + + if localEntry == nil { + // glog.V(3).Infof("dir Lookup cache miss %s", fullFilePath) + entry, err := filer_pb.GetEntry(wfs, fullFilePath) + if err != nil { + glog.V(1).Infof("dir GetEntry %s: %v", fullFilePath, err) + return fuse.ENOENT + } + localEntry = filer.FromPbEntry(string(dirPath), entry) + } else { + glog.V(4).Infof("dir Lookup cache hit %s", fullFilePath) + } + + if localEntry == nil { + return fuse.ENOENT + } + + inode := wfs.inodeToPath.Lookup(fullFilePath, localEntry.IsDirectory()) + + wfs.outputFilerEntry(out, inode, localEntry) + + return fuse.OK + +} diff --git a/weed/mount/weedfs_dir_mkrm.go b/weed/mount/weedfs_dir_mkrm.go new file mode 100644 index 000000000..839fa493b --- /dev/null +++ b/weed/mount/weedfs_dir_mkrm.go @@ -0,0 +1,111 @@ +package mount + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/hanwen/go-fuse/v2/fuse" + "os" + "strings" + "syscall" + "time" +) + +/** Create a directory + * + * Note that the mode argument may not have the type specification + * bits set, i.e. S_ISDIR(mode) can be false. To obtain the + * correct directory type bits use mode|S_IFDIR + * */ +func (wfs *WFS) Mkdir(cancel <-chan struct{}, in *fuse.MkdirIn, name string, out *fuse.EntryOut) (code fuse.Status) { + + if s := checkName(name); s != fuse.OK { + return s + } + + newEntry := &filer_pb.Entry{ + Name: name, + IsDirectory: true, + Attributes: &filer_pb.FuseAttributes{ + Mtime: time.Now().Unix(), + Crtime: time.Now().Unix(), + FileMode: uint32(os.ModeDir) | in.Mode&^uint32(wfs.option.Umask), + Uid: in.Uid, + Gid: in.Gid, + }, + } + + dirFullPath := wfs.inodeToPath.GetPath(in.NodeId) + + entryFullPath := dirFullPath.Child(name) + + err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + + wfs.mapPbIdFromLocalToFiler(newEntry) + defer wfs.mapPbIdFromFilerToLocal(newEntry) + + request := &filer_pb.CreateEntryRequest{ + Directory: string(dirFullPath), + Entry: newEntry, + Signatures: []int32{wfs.signature}, + } + + glog.V(1).Infof("mkdir: %v", request) + if err := filer_pb.CreateEntry(client, request); err != nil { + glog.V(0).Infof("mkdir %s: %v", entryFullPath, err) + return err + } + + if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil { + return fmt.Errorf("local mkdir dir %s: %v", entryFullPath, err) + } + + return nil + }) + + glog.V(0).Infof("mkdir %s: %v", entryFullPath, err) + + if err != nil { + return fuse.EIO + } + + inode := wfs.inodeToPath.Lookup(entryFullPath, true) + + wfs.outputPbEntry(out, inode, newEntry) + + return fuse.OK + +} + +/** Remove a directory */ +func (wfs *WFS) Rmdir(cancel <-chan struct{}, header *fuse.InHeader, name string) (code fuse.Status) { + + if name == "." { + return fuse.Status(syscall.EINVAL) + } + if name == ".." { + return fuse.Status(syscall.ENOTEMPTY) + } + + dirFullPath := wfs.inodeToPath.GetPath(header.NodeId) + entryFullPath := dirFullPath.Child(name) + + glog.V(3).Infof("remove directory: %v", entryFullPath) + ignoreRecursiveErr := true // ignore recursion error since the OS should manage it + err := filer_pb.Remove(wfs, string(dirFullPath), name, true, true, ignoreRecursiveErr, false, []int32{wfs.signature}) + if err != nil { + glog.V(0).Infof("remove %s: %v", entryFullPath, err) + if strings.Contains(err.Error(), filer.MsgFailDelNonEmptyFolder) { + return fuse.Status(syscall.ENOTEMPTY) + } + return fuse.ENOENT + } + + wfs.metaCache.DeleteEntry(context.Background(), entryFullPath) + wfs.inodeToPath.RemovePath(entryFullPath) + + return fuse.OK + +} diff --git a/weed/mount/weedfs_dir_read.go b/weed/mount/weedfs_dir_read.go new file mode 100644 index 000000000..ad8a161d7 --- /dev/null +++ b/weed/mount/weedfs_dir_read.go @@ -0,0 +1,132 @@ +package mount + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/mount/meta_cache" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/hanwen/go-fuse/v2/fuse" + "math" + "os" +) + +// Directory handling + +/** Open directory + * + * Unless the 'default_permissions' mount option is given, + * this method should check if opendir is permitted for this + * directory. Optionally opendir may also return an arbitrary + * filehandle in the fuse_file_info structure, which will be + * passed to readdir, releasedir and fsyncdir. + */ +func (wfs *WFS) OpenDir(cancel <-chan struct{}, input *fuse.OpenIn, out *fuse.OpenOut) (code fuse.Status) { + if !wfs.inodeToPath.HasInode(input.NodeId) { + return fuse.ENOENT + } + return fuse.OK +} + +/** Release directory + * + * If the directory has been removed after the call to opendir, the + * path parameter will be NULL. + */ +func (wfs *WFS) ReleaseDir(input *fuse.ReleaseIn) { +} + +/** Synchronize directory contents + * + * If the directory has been removed after the call to opendir, the + * path parameter will be NULL. + * + * If the datasync parameter is non-zero, then only the user data + * should be flushed, not the meta data + */ +func (wfs *WFS) FsyncDir(cancel <-chan struct{}, input *fuse.FsyncIn) (code fuse.Status) { + return fuse.OK +} + +/** Read directory + * + * The filesystem may choose between two modes of operation: + * + * 1) The readdir implementation ignores the offset parameter, and + * passes zero to the filler function's offset. The filler + * function will not return '1' (unless an error happens), so the + * whole directory is read in a single readdir operation. + * + * 2) The readdir implementation keeps track of the offsets of the + * directory entries. It uses the offset parameter and always + * passes non-zero offset to the filler function. When the buffer + * is full (or an error happens) the filler function will return + * '1'. + */ +func (wfs *WFS) ReadDir(cancel <-chan struct{}, input *fuse.ReadIn, out *fuse.DirEntryList) (code fuse.Status) { + return wfs.doReadDirectory(input, out, false) +} + +func (wfs *WFS) ReadDirPlus(cancel <-chan struct{}, input *fuse.ReadIn, out *fuse.DirEntryList) (code fuse.Status) { + return wfs.doReadDirectory(input, out, true) +} + +func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out *fuse.DirEntryList, isPlusMode bool) fuse.Status { + dirPath := wfs.inodeToPath.GetPath(input.NodeId) + + var counter uint64 + var dirEntry fuse.DirEntry + if input.Offset == 0 { + counter++ + dirEntry.Ino = input.NodeId + dirEntry.Name = "." + dirEntry.Mode = toSystemMode(os.ModeDir) + out.AddDirEntry(dirEntry) + + counter++ + parentDir, _ := dirPath.DirAndName() + parentInode := wfs.inodeToPath.GetInode(util.FullPath(parentDir)) + dirEntry.Ino = parentInode + dirEntry.Name = ".." + dirEntry.Mode = toSystemMode(os.ModeDir) + out.AddDirEntry(dirEntry) + + } + + processEachEntryFn := func(entry *filer.Entry, isLast bool) bool { + counter++ + if counter <= input.Offset { + return true + } + dirEntry.Name = entry.Name() + inode := wfs.inodeToPath.GetInode(dirPath.Child(dirEntry.Name)) + dirEntry.Ino = inode + dirEntry.Mode = toSystemMode(entry.Mode) + if !isPlusMode { + if !out.AddDirEntry(dirEntry) { + return false + } + } else { + entryOut := out.AddDirLookupEntry(dirEntry) + if entryOut == nil { + return false + } + wfs.outputFilerEntry(entryOut, inode, entry) + } + return true + } + + if err := meta_cache.EnsureVisited(wfs.metaCache, wfs, dirPath); err != nil { + glog.Errorf("dir ReadDirAll %s: %v", dirPath, err) + return fuse.EIO + } + listErr := wfs.metaCache.ListDirectoryEntries(context.Background(), dirPath, "", false, int64(math.MaxInt32), func(entry *filer.Entry) bool { + return processEachEntryFn(entry, false) + }) + if listErr != nil { + glog.Errorf("list meta cache: %v", listErr) + return fuse.EIO + } + + return fuse.OK +} diff --git a/weed/mount/weedfs_file_io.go b/weed/mount/weedfs_file_io.go new file mode 100644 index 000000000..7c8d1babc --- /dev/null +++ b/weed/mount/weedfs_file_io.go @@ -0,0 +1,98 @@ +package mount + +import ( + "github.com/hanwen/go-fuse/v2/fuse" +) + +/** + * Open a file + * + * Open flags are available in fi->flags. The following rules + * apply. + * + * - Creation (O_CREAT, O_EXCL, O_NOCTTY) flags will be + * filtered out / handled by the kernel. + * + * - Access modes (O_RDONLY, O_WRONLY, O_RDWR) should be used + * by the filesystem to check if the operation is + * permitted. If the ``-o default_permissions`` mount + * option is given, this check is already done by the + * kernel before calling open() and may thus be omitted by + * the filesystem. + * + * - When writeback caching is enabled, the kernel may send + * read requests even for files opened with O_WRONLY. The + * filesystem should be prepared to handle this. + * + * - When writeback caching is disabled, the filesystem is + * expected to properly handle the O_APPEND flag and ensure + * that each write is appending to the end of the file. + * + * - When writeback caching is enabled, the kernel will + * handle O_APPEND. However, unless all changes to the file + * come through the kernel this will not work reliably. The + * filesystem should thus either ignore the O_APPEND flag + * (and let the kernel handle it), or return an error + * (indicating that reliably O_APPEND is not available). + * + * Filesystem may store an arbitrary file handle (pointer, + * index, etc) in fi->fh, and use this in other all other file + * operations (read, write, flush, release, fsync). + * + * Filesystem may also implement stateless file I/O and not store + * anything in fi->fh. + * + * There are also some flags (direct_io, keep_cache) which the + * filesystem may set in fi, to change the way the file is opened. + * See fuse_file_info structure in <fuse_common.h> for more details. + * + * If this request is answered with an error code of ENOSYS + * and FUSE_CAP_NO_OPEN_SUPPORT is set in + * `fuse_conn_info.capable`, this is treated as success and + * future calls to open and release will also succeed without being + * sent to the filesystem process. + * + * Valid replies: + * fuse_reply_open + * fuse_reply_err + * + * @param req request handle + * @param ino the inode number + * @param fi file information +*/ +func (wfs *WFS) Open(cancel <-chan struct{}, in *fuse.OpenIn, out *fuse.OpenOut) (status fuse.Status) { + fileHandle, code := wfs.AcquireHandle(in.NodeId, in.Uid, in.Gid) + if code == fuse.OK { + out.Fh = uint64(fileHandle.fh) + } + return code +} + +/** + * Release an open file + * + * Release is called when there are no more references to an open + * file: all file descriptors are closed and all memory mappings + * are unmapped. + * + * For every open call there will be exactly one release call (unless + * the filesystem is force-unmounted). + * + * The filesystem may reply with an error, but error values are + * not returned to close() or munmap() which triggered the + * release. + * + * fi->fh will contain the value set by the open method, or will + * be undefined if the open method didn't set any value. + * fi->flags will contain the same flags as for open. + * + * Valid replies: + * fuse_reply_err + * + * @param req request handle + * @param ino the inode number + * @param fi file information + */ +func (wfs *WFS) Release(cancel <-chan struct{}, in *fuse.ReleaseIn) { + wfs.ReleaseHandle(FileHandleId(in.Fh)) +} diff --git a/weed/mount/weedfs_file_mkrm.go b/weed/mount/weedfs_file_mkrm.go new file mode 100644 index 000000000..c3fd04661 --- /dev/null +++ b/weed/mount/weedfs_file_mkrm.go @@ -0,0 +1,130 @@ +package mount + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/hanwen/go-fuse/v2/fuse" + "time" +) + +/** + * Create and open a file + * + * If the file does not exist, first create it with the specified + * mode, and then open it. + * + * If this method is not implemented or under Linux kernel + * versions earlier than 2.6.15, the mknod() and open() methods + * will be called instead. + */ +func (wfs *WFS) Create(cancel <-chan struct{}, in *fuse.CreateIn, name string, out *fuse.CreateOut) (code fuse.Status) { + // if implemented, need to use + // inode := wfs.inodeToPath.Lookup(entryFullPath) + // to ensure nlookup counter + return fuse.ENOSYS +} + +/** Create a file node + * + * This is called for creation of all non-directory, non-symlink + * nodes. If the filesystem defines a create() method, then for + * regular files that will be called instead. + */ +func (wfs *WFS) Mknod(cancel <-chan struct{}, in *fuse.MknodIn, name string, out *fuse.EntryOut) (code fuse.Status) { + + if s := checkName(name); s != fuse.OK { + return s + } + + newEntry := &filer_pb.Entry{ + Name: name, + IsDirectory: false, + Attributes: &filer_pb.FuseAttributes{ + Mtime: time.Now().Unix(), + Crtime: time.Now().Unix(), + FileMode: uint32(toFileMode(in.Mode) &^ wfs.option.Umask), + Uid: in.Uid, + Gid: in.Gid, + Collection: wfs.option.Collection, + Replication: wfs.option.Replication, + TtlSec: wfs.option.TtlSec, + }, + } + + dirFullPath := wfs.inodeToPath.GetPath(in.NodeId) + + entryFullPath := dirFullPath.Child(name) + + err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + + wfs.mapPbIdFromLocalToFiler(newEntry) + defer wfs.mapPbIdFromFilerToLocal(newEntry) + + request := &filer_pb.CreateEntryRequest{ + Directory: string(dirFullPath), + Entry: newEntry, + Signatures: []int32{wfs.signature}, + } + + glog.V(1).Infof("mknod: %v", request) + if err := filer_pb.CreateEntry(client, request); err != nil { + glog.V(0).Infof("mknod %s: %v", entryFullPath, err) + return err + } + + if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil { + return fmt.Errorf("local mknod %s: %v", entryFullPath, err) + } + + return nil + }) + + glog.V(0).Infof("mknod %s: %v", entryFullPath, err) + + if err != nil { + return fuse.EIO + } + + inode := wfs.inodeToPath.Lookup(entryFullPath, false) + + wfs.outputPbEntry(out, inode, newEntry) + + return fuse.OK + +} + +/** Remove a file */ +func (wfs *WFS) Unlink(cancel <-chan struct{}, header *fuse.InHeader, name string) (code fuse.Status) { + + dirFullPath := wfs.inodeToPath.GetPath(header.NodeId) + entryFullPath := dirFullPath.Child(name) + + entry, status := wfs.maybeLoadEntry(entryFullPath) + if status != fuse.OK { + return status + } + + // first, ensure the filer store can correctly delete + glog.V(3).Infof("remove file: %v", entryFullPath) + isDeleteData := entry != nil && entry.HardLinkCounter <= 1 + err := filer_pb.Remove(wfs, string(dirFullPath), name, isDeleteData, false, false, false, []int32{wfs.signature}) + if err != nil { + glog.V(0).Infof("remove %s: %v", entryFullPath, err) + return fuse.ENOENT + } + + // then, delete meta cache + if err = wfs.metaCache.DeleteEntry(context.Background(), entryFullPath); err != nil { + glog.V(3).Infof("local DeleteEntry %s: %v", entryFullPath, err) + return fuse.EIO + } + + wfs.metaCache.DeleteEntry(context.Background(), entryFullPath) + wfs.inodeToPath.RemovePath(entryFullPath) + + return fuse.OK + +} diff --git a/weed/mount/weedfs_file_read.go b/weed/mount/weedfs_file_read.go new file mode 100644 index 000000000..00143a5b4 --- /dev/null +++ b/weed/mount/weedfs_file_read.go @@ -0,0 +1,59 @@ +package mount + +import ( + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/hanwen/go-fuse/v2/fuse" + "io" +) + +/** + * Read data + * + * Read should send exactly the number of bytes requested except + * on EOF or error, otherwise the rest of the data will be + * substituted with zeroes. An exception to this is when the file + * has been opened in 'direct_io' mode, in which case the return + * value of the read system call will reflect the return value of + * this operation. + * + * fi->fh will contain the value set by the open method, or will + * be undefined if the open method didn't set any value. + * + * Valid replies: + * fuse_reply_buf + * fuse_reply_iov + * fuse_reply_data + * fuse_reply_err + * + * @param req request handle + * @param ino the inode number + * @param size number of bytes to read + * @param off offset to read from + * @param fi file information + */ +func (wfs *WFS) Read(cancel <-chan struct{}, in *fuse.ReadIn, buff []byte) (fuse.ReadResult, fuse.Status) { + fh := wfs.GetHandle(FileHandleId(in.Fh)) + if fh == nil { + return nil, fuse.ENOENT + } + + offset := int64(in.Offset) + fh.lockForRead(offset, len(buff)) + defer fh.unlockForRead(offset, len(buff)) + + totalRead, err := fh.readFromChunks(buff, offset) + if err == nil || err == io.EOF { + maxStop := fh.readFromDirtyPages(buff, offset) + totalRead = max(maxStop-offset, totalRead) + } + if err == io.EOF { + err = nil + } + + if err != nil { + glog.Warningf("file handle read %s %d: %v", fh.FullPath(), totalRead, err) + return nil, fuse.EIO + } + + return fuse.ReadResultData(buff[:totalRead]), fuse.OK +} diff --git a/weed/mount/weedfs_file_sync.go b/weed/mount/weedfs_file_sync.go new file mode 100644 index 000000000..29a13690b --- /dev/null +++ b/weed/mount/weedfs_file_sync.go @@ -0,0 +1,180 @@ +package mount + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/hanwen/go-fuse/v2/fuse" + "os" + "time" +) + +/** + * Flush method + * + * This is called on each close() of the opened file. + * + * Since file descriptors can be duplicated (dup, dup2, fork), for + * one open call there may be many flush calls. + * + * Filesystems shouldn't assume that flush will always be called + * after some writes, or that if will be called at all. + * + * fi->fh will contain the value set by the open method, or will + * be undefined if the open method didn't set any value. + * + * NOTE: the name of the method is misleading, since (unlike + * fsync) the filesystem is not forced to flush pending writes. + * One reason to flush data is if the filesystem wants to return + * write errors during close. However, such use is non-portable + * because POSIX does not require [close] to wait for delayed I/O to + * complete. + * + * If the filesystem supports file locking operations (setlk, + * getlk) it should remove all locks belonging to 'fi->owner'. + * + * If this request is answered with an error code of ENOSYS, + * this is treated as success and future calls to flush() will + * succeed automatically without being send to the filesystem + * process. + * + * Valid replies: + * fuse_reply_err + * + * @param req request handle + * @param ino the inode number + * @param fi file information + * + * [close]: http://pubs.opengroup.org/onlinepubs/9699919799/functions/close.html + */ +func (wfs *WFS) Flush(cancel <-chan struct{}, in *fuse.FlushIn) fuse.Status { + fh := wfs.GetHandle(FileHandleId(in.Fh)) + if fh == nil { + return fuse.ENOENT + } + + fh.Lock() + defer fh.Unlock() + + return wfs.doFlush(fh, in.Uid, in.Gid) +} + +/** + * Synchronize file contents + * + * If the datasync parameter is non-zero, then only the user data + * should be flushed, not the meta data. + * + * If this request is answered with an error code of ENOSYS, + * this is treated as success and future calls to fsync() will + * succeed automatically without being send to the filesystem + * process. + * + * Valid replies: + * fuse_reply_err + * + * @param req request handle + * @param ino the inode number + * @param datasync flag indicating if only data should be flushed + * @param fi file information + */ +func (wfs *WFS) Fsync(cancel <-chan struct{}, in *fuse.FsyncIn) (code fuse.Status) { + + fh := wfs.GetHandle(FileHandleId(in.Fh)) + if fh == nil { + return fuse.ENOENT + } + + fh.Lock() + defer fh.Unlock() + + return wfs.doFlush(fh, in.Uid, in.Gid) + +} + +func (wfs *WFS) doFlush(fh *FileHandle, uid, gid uint32) fuse.Status { + // flush works at fh level + fileFullPath := fh.FullPath() + dir, _ := fileFullPath.DirAndName() + // send the data to the OS + glog.V(4).Infof("doFlush %s fh %d", fileFullPath, fh.handle) + + if err := fh.dirtyPages.FlushData(); err != nil { + glog.Errorf("%v doFlush: %v", fileFullPath, err) + return fuse.EIO + } + + if !fh.dirtyMetadata { + return fuse.OK + } + + err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + + entry := fh.entry + if entry == nil { + return nil + } + + if entry.Attributes != nil { + entry.Attributes.Mime = fh.contentType + if entry.Attributes.Uid == 0 { + entry.Attributes.Uid = uid + } + if entry.Attributes.Gid == 0 { + entry.Attributes.Gid = gid + } + if entry.Attributes.Crtime == 0 { + entry.Attributes.Crtime = time.Now().Unix() + } + entry.Attributes.Mtime = time.Now().Unix() + entry.Attributes.FileMode = uint32(os.FileMode(entry.Attributes.FileMode) &^ wfs.option.Umask) + entry.Attributes.Collection, entry.Attributes.Replication = fh.dirtyPages.GetStorageOptions() + } + + request := &filer_pb.CreateEntryRequest{ + Directory: string(dir), + Entry: entry, + Signatures: []int32{wfs.signature}, + } + + glog.V(4).Infof("%s set chunks: %v", fileFullPath, len(entry.Chunks)) + for i, chunk := range entry.Chunks { + glog.V(4).Infof("%s chunks %d: %v [%d,%d)", fileFullPath, i, chunk.GetFileIdString(), chunk.Offset, chunk.Offset+int64(chunk.Size)) + } + + manifestChunks, nonManifestChunks := filer.SeparateManifestChunks(entry.Chunks) + + chunks, _ := filer.CompactFileChunks(wfs.LookupFn(), nonManifestChunks) + chunks, manifestErr := filer.MaybeManifestize(wfs.saveDataAsChunk(fileFullPath), chunks) + if manifestErr != nil { + // not good, but should be ok + glog.V(0).Infof("MaybeManifestize: %v", manifestErr) + } + entry.Chunks = append(chunks, manifestChunks...) + + wfs.mapPbIdFromLocalToFiler(request.Entry) + defer wfs.mapPbIdFromFilerToLocal(request.Entry) + + if err := filer_pb.CreateEntry(client, request); err != nil { + glog.Errorf("fh flush create %s: %v", fileFullPath, err) + return fmt.Errorf("fh flush create %s: %v", fileFullPath, err) + } + + wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)) + + return nil + }) + + if err == nil { + fh.dirtyMetadata = false + } + + if err != nil { + glog.Errorf("%v fh %d flush: %v", fileFullPath, fh.handle, err) + return fuse.EIO + } + + return fuse.OK +} diff --git a/weed/mount/weedfs_file_write.go b/weed/mount/weedfs_file_write.go new file mode 100644 index 000000000..efdf39386 --- /dev/null +++ b/weed/mount/weedfs_file_write.go @@ -0,0 +1,66 @@ +package mount + +import ( + "github.com/hanwen/go-fuse/v2/fuse" + "net/http" +) + +/** + * Write data + * + * Write should return exactly the number of bytes requested + * except on error. An exception to this is when the file has + * been opened in 'direct_io' mode, in which case the return value + * of the write system call will reflect the return value of this + * operation. + * + * Unless FUSE_CAP_HANDLE_KILLPRIV is disabled, this method is + * expected to reset the setuid and setgid bits. + * + * fi->fh will contain the value set by the open method, or will + * be undefined if the open method didn't set any value. + * + * Valid replies: + * fuse_reply_write + * fuse_reply_err + * + * @param req request handle + * @param ino the inode number + * @param buf data to write + * @param size number of bytes to write + * @param off offset to write to + * @param fi file information + */ +func (wfs *WFS) Write(cancel <-chan struct{}, in *fuse.WriteIn, data []byte) (written uint32, code fuse.Status) { + + fh := wfs.GetHandle(FileHandleId(in.Fh)) + if fh == nil { + return 0, fuse.ENOENT + } + + fh.Lock() + defer fh.Unlock() + + entry := fh.entry + if entry == nil { + return 0, fuse.OK + } + + entry.Content = nil + offset := int64(in.Offset) + entry.Attributes.FileSize = uint64(max(offset+int64(len(data)), int64(entry.Attributes.FileSize))) + // glog.V(4).Infof("%v write [%d,%d) %d", fh.f.fullpath(), req.Offset, req.Offset+int64(len(req.Data)), len(req.Data)) + + fh.dirtyPages.AddPage(offset, data) + + written = uint32(len(data)) + + if offset == 0 { + // detect mime type + fh.contentType = http.DetectContentType(data) + } + + fh.dirtyMetadata = true + + return written, fuse.OK +} diff --git a/weed/mount/weedfs_filehandle.go b/weed/mount/weedfs_filehandle.go new file mode 100644 index 000000000..3e085df37 --- /dev/null +++ b/weed/mount/weedfs_filehandle.go @@ -0,0 +1,20 @@ +package mount + +import "github.com/hanwen/go-fuse/v2/fuse" + +func (wfs *WFS) AcquireHandle(inode uint64, uid, gid uint32) (fileHandle *FileHandle, code fuse.Status) { + _, _, entry, status := wfs.maybeReadEntry(inode) + if status == fuse.OK { + fileHandle = wfs.fhmap.AcquireFileHandle(wfs, inode, entry) + fileHandle.entry = entry + } + return +} + +func (wfs *WFS) ReleaseHandle(handleId FileHandleId) { + wfs.fhmap.ReleaseByHandle(handleId) +} + +func (wfs *WFS) GetHandle(handleId FileHandleId) *FileHandle { + return wfs.fhmap.GetFileHandle(handleId) +} diff --git a/weed/mount/weedfs_forget.go b/weed/mount/weedfs_forget.go new file mode 100644 index 000000000..62946b216 --- /dev/null +++ b/weed/mount/weedfs_forget.go @@ -0,0 +1,68 @@ +package mount + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/util" +) + +// Forget is called when the kernel discards entries from its +// dentry cache. This happens on unmount, and when the kernel +// is short on memory. Since it is not guaranteed to occur at +// any moment, and since there is no return value, Forget +// should not do I/O, as there is no channel to report back +// I/O errors. +// from https://github.com/libfuse/libfuse/blob/master/include/fuse_lowlevel.h +/** + * Forget about an inode + * + * This function is called when the kernel removes an inode + * from its internal caches. + * + * The inode's lookup count increases by one for every call to + * fuse_reply_entry and fuse_reply_create. The nlookup parameter + * indicates by how much the lookup count should be decreased. + * + * Inodes with a non-zero lookup count may receive request from + * the kernel even after calls to unlink, rmdir or (when + * overwriting an existing file) rename. Filesystems must handle + * such requests properly and it is recommended to defer removal + * of the inode until the lookup count reaches zero. Calls to + * unlink, rmdir or rename will be followed closely by forget + * unless the file or directory is open, in which case the + * kernel issues forget only after the release or releasedir + * calls. + * + * Note that if a file system will be exported over NFS the + * inodes lifetime must extend even beyond forget. See the + * generation field in struct fuse_entry_param above. + * + * On unmount the lookup count for all inodes implicitly drops + * to zero. It is not guaranteed that the file system will + * receive corresponding forget messages for the affected + * inodes. + * + * Valid replies: + * fuse_reply_none + * + * @param req request handle + * @param ino the inode number + * @param nlookup the number of lookups to forget + */ +/* +https://libfuse.github.io/doxygen/include_2fuse__lowlevel_8h.html + +int fuse_reply_entry ( fuse_req_t req, +const struct fuse_entry_param * e +) +Reply with a directory entry + +Possible requests: lookup, mknod, mkdir, symlink, link + +Side effects: increments the lookup count on success + +*/ +func (wfs *WFS) Forget(nodeid, nlookup uint64) { + wfs.inodeToPath.Forget(nodeid, nlookup, func(dir util.FullPath) { + wfs.metaCache.DeleteFolderChildren(context.Background(), dir) + }) +} diff --git a/weed/mount/weedfs_link.go b/weed/mount/weedfs_link.go new file mode 100644 index 000000000..ca252d639 --- /dev/null +++ b/weed/mount/weedfs_link.go @@ -0,0 +1,93 @@ +package mount + +import ( + "context" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/hanwen/go-fuse/v2/fuse" + "time" +) + +const ( + HARD_LINK_MARKER = '\x01' +) + +/** Create a hard link to a file */ +func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *fuse.EntryOut) (code fuse.Status) { + + if s := checkName(name); s != fuse.OK { + return s + } + + newParentPath := wfs.inodeToPath.GetPath(in.NodeId) + oldEntryPath := wfs.inodeToPath.GetPath(in.Oldnodeid) + oldParentPath, _ := oldEntryPath.DirAndName() + + oldEntry, status := wfs.maybeLoadEntry(oldEntryPath) + if status != fuse.OK { + return status + } + + // update old file to hardlink mode + if len(oldEntry.HardLinkId) == 0 { + oldEntry.HardLinkId = append(util.RandomBytes(16), HARD_LINK_MARKER) + oldEntry.HardLinkCounter = 1 + } + oldEntry.HardLinkCounter++ + updateOldEntryRequest := &filer_pb.UpdateEntryRequest{ + Directory: oldParentPath, + Entry: oldEntry, + Signatures: []int32{wfs.signature}, + } + + // CreateLink 1.2 : update new file to hardlink mode + oldEntry.Attributes.Mtime = time.Now().Unix() + request := &filer_pb.CreateEntryRequest{ + Directory: string(newParentPath), + Entry: &filer_pb.Entry{ + Name: name, + IsDirectory: false, + Attributes: oldEntry.Attributes, + Chunks: oldEntry.Chunks, + Extended: oldEntry.Extended, + HardLinkId: oldEntry.HardLinkId, + HardLinkCounter: oldEntry.HardLinkCounter, + }, + Signatures: []int32{wfs.signature}, + } + + // apply changes to the filer, and also apply to local metaCache + err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + + wfs.mapPbIdFromLocalToFiler(request.Entry) + defer wfs.mapPbIdFromFilerToLocal(request.Entry) + + if err := filer_pb.UpdateEntry(client, updateOldEntryRequest); err != nil { + return err + } + wfs.metaCache.UpdateEntry(context.Background(), filer.FromPbEntry(updateOldEntryRequest.Directory, updateOldEntryRequest.Entry)) + + if err := filer_pb.CreateEntry(client, request); err != nil { + return err + } + + wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)) + + return nil + }) + + newEntryPath := newParentPath.Child(name) + + if err != nil { + glog.V(0).Infof("Link %v -> %s: %v", oldEntryPath, newEntryPath, err) + return fuse.EIO + } + + inode := wfs.inodeToPath.Lookup(newEntryPath, false) + + wfs.outputPbEntry(out, inode, request.Entry) + + return fuse.OK +} diff --git a/weed/mount/weedfs_rename.go b/weed/mount/weedfs_rename.go new file mode 100644 index 000000000..9e461abce --- /dev/null +++ b/weed/mount/weedfs_rename.go @@ -0,0 +1,235 @@ +package mount + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/hanwen/go-fuse/v2/fs" + "github.com/hanwen/go-fuse/v2/fuse" + "io" + "strings" + "syscall" +) + +/** Rename a file + * + * If the target exists it should be atomically replaced. If + * the target's inode's lookup count is non-zero, the file + * system is expected to postpone any removal of the inode + * until the lookup count reaches zero (see description of the + * forget function). + * + * If this request is answered with an error code of ENOSYS, this is + * treated as a permanent failure with error code EINVAL, i.e. all + * future bmap requests will fail with EINVAL without being + * send to the filesystem process. + * + * *flags* may be `RENAME_EXCHANGE` or `RENAME_NOREPLACE`. If + * RENAME_NOREPLACE is specified, the filesystem must not + * overwrite *newname* if it exists and return an error + * instead. If `RENAME_EXCHANGE` is specified, the filesystem + * must atomically exchange the two files, i.e. both must + * exist and neither may be deleted. + * + * Valid replies: + * fuse_reply_err + * + * @param req request handle + * @param parent inode number of the old parent directory + * @param name old name + * @param newparent inode number of the new parent directory + * @param newname new name + */ +/* +renameat2() + renameat2() has an additional flags argument. A renameat2() call + with a zero flags argument is equivalent to renameat(). + + The flags argument is a bit mask consisting of zero or more of + the following flags: + + RENAME_EXCHANGE + Atomically exchange oldpath and newpath. Both pathnames + must exist but may be of different types (e.g., one could + be a non-empty directory and the other a symbolic link). + + RENAME_NOREPLACE + Don't overwrite newpath of the rename. Return an error if + newpath already exists. + + RENAME_NOREPLACE can't be employed together with + RENAME_EXCHANGE. + + RENAME_NOREPLACE requires support from the underlying + filesystem. Support for various filesystems was added as + follows: + + * ext4 (Linux 3.15); + + * btrfs, tmpfs, and cifs (Linux 3.17); + + * xfs (Linux 4.0); + + * Support for many other filesystems was added in Linux + 4.9, including ext2, minix, reiserfs, jfs, vfat, and + bpf. + + RENAME_WHITEOUT (since Linux 3.18) + This operation makes sense only for overlay/union + filesystem implementations. + + Specifying RENAME_WHITEOUT creates a "whiteout" object at + the source of the rename at the same time as performing + the rename. The whole operation is atomic, so that if the + rename succeeds then the whiteout will also have been + created. + + A "whiteout" is an object that has special meaning in + union/overlay filesystem constructs. In these constructs, + multiple layers exist and only the top one is ever + modified. A whiteout on an upper layer will effectively + hide a matching file in the lower layer, making it appear + as if the file didn't exist. + + When a file that exists on the lower layer is renamed, the + file is first copied up (if not already on the upper + layer) and then renamed on the upper, read-write layer. + At the same time, the source file needs to be "whiteouted" + (so that the version of the source file in the lower layer + is rendered invisible). The whole operation needs to be + done atomically. + + When not part of a union/overlay, the whiteout appears as + a character device with a {0,0} device number. (Note that + other union/overlay implementations may employ different + methods for storing whiteout entries; specifically, BSD + union mount employs a separate inode type, DT_WHT, which, + while supported by some filesystems available in Linux, + such as CODA and XFS, is ignored by the kernel's whiteout + support code, as of Linux 4.19, at least.) + + RENAME_WHITEOUT requires the same privileges as creating a + device node (i.e., the CAP_MKNOD capability). + + RENAME_WHITEOUT can't be employed together with + RENAME_EXCHANGE. + + RENAME_WHITEOUT requires support from the underlying + filesystem. Among the filesystems that support it are + tmpfs (since Linux 3.18), ext4 (since Linux 3.18), XFS + (since Linux 4.1), f2fs (since Linux 4.2), btrfs (since + Linux 4.7), and ubifs (since Linux 4.9). +*/ +const ( + RenameEmptyFlag = 0 + RenameNoReplace = 1 + RenameExchange = fs.RENAME_EXCHANGE + RenameWhiteout = 3 +) + +func (wfs *WFS) Rename(cancel <-chan struct{}, in *fuse.RenameIn, oldName string, newName string) (code fuse.Status) { + if s := checkName(newName); s != fuse.OK { + return s + } + + switch in.Flags { + case RenameEmptyFlag: + case RenameNoReplace: + case RenameExchange: + case RenameWhiteout: + return fuse.ENOTSUP + default: + return fuse.EINVAL + } + + oldDir := wfs.inodeToPath.GetPath(in.NodeId) + oldPath := oldDir.Child(oldName) + newDir := wfs.inodeToPath.GetPath(in.Newdir) + newPath := newDir.Child(newName) + + glog.V(4).Infof("dir Rename %s => %s", oldPath, newPath) + + // update remote filer + err := wfs.WithFilerClient(true, func(client filer_pb.SeaweedFilerClient) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + request := &filer_pb.StreamRenameEntryRequest{ + OldDirectory: string(oldDir), + OldName: oldName, + NewDirectory: string(newDir), + NewName: newName, + Signatures: []int32{wfs.signature}, + } + + stream, err := client.StreamRenameEntry(ctx, request) + if err != nil { + code = fuse.EIO + return fmt.Errorf("dir AtomicRenameEntry %s => %s : %v", oldPath, newPath, err) + } + + for { + resp, recvErr := stream.Recv() + if recvErr != nil { + if recvErr == io.EOF { + break + } else { + if strings.Contains(recvErr.Error(), "not empty") { + code = fuse.Status(syscall.ENOTEMPTY) + } else if strings.Contains(recvErr.Error(), "not directory") { + code = fuse.ENOTDIR + } + return fmt.Errorf("dir Rename %s => %s receive: %v", oldPath, newPath, recvErr) + } + } + + if err = wfs.handleRenameResponse(ctx, resp); err != nil { + glog.V(0).Infof("dir Rename %s => %s : %v", oldPath, newPath, err) + return err + } + + } + + return nil + + }) + if err != nil { + glog.V(0).Infof("Link: %v", err) + return + } + + return fuse.OK + +} + +func (wfs *WFS) handleRenameResponse(ctx context.Context, resp *filer_pb.StreamRenameEntryResponse) error { + // comes from filer StreamRenameEntry, can only be create or delete entry + + if resp.EventNotification.NewEntry != nil { + // with new entry, the old entry name also exists. This is the first step to create new entry + newEntry := filer.FromPbEntry(resp.EventNotification.NewParentPath, resp.EventNotification.NewEntry) + if err := wfs.metaCache.AtomicUpdateEntryFromFiler(ctx, "", newEntry); err != nil { + return err + } + + oldParent, newParent := util.FullPath(resp.Directory), util.FullPath(resp.EventNotification.NewParentPath) + oldName, newName := resp.EventNotification.OldEntry.Name, resp.EventNotification.NewEntry.Name + + oldPath := oldParent.Child(oldName) + newPath := newParent.Child(newName) + + wfs.inodeToPath.MovePath(oldPath, newPath) + + } else if resp.EventNotification.OldEntry != nil { + // without new entry, only old entry name exists. This is the second step to delete old entry + if err := wfs.metaCache.AtomicUpdateEntryFromFiler(ctx, util.NewFullPath(resp.Directory, resp.EventNotification.OldEntry.Name), nil); err != nil { + return err + } + } + + return nil + +} diff --git a/weed/mount/weedfs_stats.go b/weed/mount/weedfs_stats.go new file mode 100644 index 000000000..0da41ab0b --- /dev/null +++ b/weed/mount/weedfs_stats.go @@ -0,0 +1,80 @@ +package mount + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/hanwen/go-fuse/v2/fuse" + "math" + "time" +) + +const blockSize = 512 + +type statsCache struct { + filer_pb.StatisticsResponse + lastChecked int64 // unix time in seconds +} + +func (wfs *WFS) StatFs(cancel <-chan struct{}, in *fuse.InHeader, out *fuse.StatfsOut) (code fuse.Status) { + + // glog.V(4).Infof("reading fs stats") + + if wfs.stats.lastChecked < time.Now().Unix()-20 { + + err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + + request := &filer_pb.StatisticsRequest{ + Collection: wfs.option.Collection, + Replication: wfs.option.Replication, + Ttl: fmt.Sprintf("%ds", wfs.option.TtlSec), + DiskType: string(wfs.option.DiskType), + } + + glog.V(4).Infof("reading filer stats: %+v", request) + resp, err := client.Statistics(context.Background(), request) + if err != nil { + glog.V(0).Infof("reading filer stats %v: %v", request, err) + return err + } + glog.V(4).Infof("read filer stats: %+v", resp) + + wfs.stats.TotalSize = resp.TotalSize + wfs.stats.UsedSize = resp.UsedSize + wfs.stats.FileCount = resp.FileCount + wfs.stats.lastChecked = time.Now().Unix() + + return nil + }) + if err != nil { + glog.V(0).Infof("filer Statistics: %v", err) + return fuse.OK + } + } + + totalDiskSize := wfs.stats.TotalSize + usedDiskSize := wfs.stats.UsedSize + actualFileCount := wfs.stats.FileCount + + // Compute the total number of available blocks + out.Blocks = totalDiskSize / blockSize + + // Compute the number of used blocks + numBlocks := uint64(usedDiskSize / blockSize) + + // Report the number of free and available blocks for the block size + out.Bfree = out.Blocks - numBlocks + out.Bavail = out.Blocks - numBlocks + out.Bsize = uint32(blockSize) + + // Report the total number of possible files in the file system (and those free) + out.Files = math.MaxInt64 + out.Ffree = math.MaxInt64 - actualFileCount + + // Report the maximum length of a name and the minimum fragment size + out.NameLen = 1024 + out.Frsize = uint32(blockSize) + + return fuse.OK +} diff --git a/weed/mount/weedfs_symlink.go b/weed/mount/weedfs_symlink.go new file mode 100644 index 000000000..c47ad0a2e --- /dev/null +++ b/weed/mount/weedfs_symlink.go @@ -0,0 +1,78 @@ +package mount + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/hanwen/go-fuse/v2/fuse" + "os" + "time" +) + +/** Create a symbolic link */ +func (wfs *WFS) Symlink(cancel <-chan struct{}, header *fuse.InHeader, target string, name string, out *fuse.EntryOut) (code fuse.Status) { + + if s := checkName(name); s != fuse.OK { + return s + } + + dirPath := wfs.inodeToPath.GetPath(header.NodeId) + entryFullPath := dirPath.Child(name) + + request := &filer_pb.CreateEntryRequest{ + Directory: string(dirPath), + Entry: &filer_pb.Entry{ + Name: name, + IsDirectory: false, + Attributes: &filer_pb.FuseAttributes{ + Mtime: time.Now().Unix(), + Crtime: time.Now().Unix(), + FileMode: uint32((os.FileMode(0777) | os.ModeSymlink) &^ wfs.option.Umask), + Uid: header.Uid, + Gid: header.Gid, + SymlinkTarget: target, + }, + }, + Signatures: []int32{wfs.signature}, + } + + err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + + wfs.mapPbIdFromLocalToFiler(request.Entry) + defer wfs.mapPbIdFromFilerToLocal(request.Entry) + + if err := filer_pb.CreateEntry(client, request); err != nil { + return fmt.Errorf("symlink %s: %v", entryFullPath, err) + } + + wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)) + + return nil + }) + if err != nil { + glog.V(0).Infof("Symlink %s => %s: %v", entryFullPath, target, err) + return fuse.EIO + } + + inode := wfs.inodeToPath.Lookup(entryFullPath, false) + + wfs.outputPbEntry(out, inode, request.Entry) + + return fuse.OK +} + +func (wfs *WFS) Readlink(cancel <-chan struct{}, header *fuse.InHeader) (out []byte, code fuse.Status) { + entryFullPath := wfs.inodeToPath.GetPath(header.NodeId) + + entry, status := wfs.maybeLoadEntry(entryFullPath) + if status != fuse.OK { + return nil, status + } + if os.FileMode(entry.Attributes.FileMode)&os.ModeSymlink == 0 { + return nil, fuse.EINVAL + } + + return []byte(entry.Attributes.SymlinkTarget), fuse.OK +} diff --git a/weed/mount/weedfs_unsupported.go b/weed/mount/weedfs_unsupported.go new file mode 100644 index 000000000..2536811b8 --- /dev/null +++ b/weed/mount/weedfs_unsupported.go @@ -0,0 +1,65 @@ +package mount + +import "github.com/hanwen/go-fuse/v2/fuse" + +// https://github.com/libfuse/libfuse/blob/48ae2e72b39b6a31cb2194f6f11786b7ca06aac6/include/fuse.h#L778 + +/** + * Copy a range of data from one file to anotherNiels de Vos, 4 years ago: • libfuse: add copy_file_range() support + * + * Performs an optimized copy between two file descriptors without the + * additional cost of transferring data through the FUSE kernel module + * to user space (glibc) and then back into the FUSE filesystem again. + * + * In case this method is not implemented, applications are expected to + * fall back to a regular file copy. (Some glibc versions did this + * emulation automatically, but the emulation has been removed from all + * glibc release branches.) + */ +func (wfs *WFS) CopyFileRange(cancel <-chan struct{}, in *fuse.CopyFileRangeIn) (written uint32, code fuse.Status) { + return 0, fuse.ENOSYS +} + +/** + * Allocates space for an open file + * + * This function ensures that required space is allocated for specified + * file. If this function returns success then any subsequent write + * request to specified range is guaranteed not to fail because of lack + * of space on the file system media. + */ +func (wfs *WFS) Fallocate(cancel <-chan struct{}, in *fuse.FallocateIn) (code fuse.Status) { + return fuse.ENOSYS +} + +/** + * Find next data or hole after the specified offset + */ +func (wfs *WFS) Lseek(cancel <-chan struct{}, in *fuse.LseekIn, out *fuse.LseekOut) fuse.Status { + return fuse.ENOSYS +} + +func (wfs *WFS) GetLk(cancel <-chan struct{}, in *fuse.LkIn, out *fuse.LkOut) (code fuse.Status) { + return fuse.ENOSYS +} + +func (wfs *WFS) SetLk(cancel <-chan struct{}, in *fuse.LkIn) (code fuse.Status) { + return fuse.ENOSYS +} + +func (wfs *WFS) SetLkw(cancel <-chan struct{}, in *fuse.LkIn) (code fuse.Status) { + return fuse.ENOSYS +} + +/** + * Check file access permissions + * + * This will be called for the access() system call. If the + * 'default_permissions' mount option is given, this method is not + * called. + * + * This method is not called under Linux kernel versions 2.4.x + */ +func (wfs *WFS) Access(cancel <-chan struct{}, input *fuse.AccessIn) (code fuse.Status) { + return fuse.ENOSYS +} diff --git a/weed/mount/weedfs_write.go b/weed/mount/weedfs_write.go new file mode 100644 index 000000000..723ce9c34 --- /dev/null +++ b/weed/mount/weedfs_write.go @@ -0,0 +1,84 @@ +package mount + +import ( + "context" + "fmt" + "io" + + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/operation" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/security" + "github.com/chrislusf/seaweedfs/weed/util" +) + +func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath) filer.SaveDataAsChunkFunctionType { + + return func(reader io.Reader, filename string, offset int64) (chunk *filer_pb.FileChunk, collection, replication string, err error) { + var fileId, host string + var auth security.EncodedJwt + + if err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + return util.Retry("assignVolume", func() error { + request := &filer_pb.AssignVolumeRequest{ + Count: 1, + Replication: wfs.option.Replication, + Collection: wfs.option.Collection, + TtlSec: wfs.option.TtlSec, + DiskType: string(wfs.option.DiskType), + DataCenter: wfs.option.DataCenter, + Path: string(fullPath), + } + + resp, err := client.AssignVolume(context.Background(), request) + if err != nil { + glog.V(0).Infof("assign volume failure %v: %v", request, err) + return err + } + if resp.Error != "" { + return fmt.Errorf("assign volume failure %v: %v", request, resp.Error) + } + + fileId, auth = resp.FileId, security.EncodedJwt(resp.Auth) + loc := resp.Location + host = wfs.AdjustedUrl(loc) + collection, replication = resp.Collection, resp.Replication + + return nil + }) + }); err != nil { + return nil, "", "", fmt.Errorf("filerGrpcAddress assign volume: %v", err) + } + + fileUrl := fmt.Sprintf("http://%s/%s", host, fileId) + if wfs.option.VolumeServerAccess == "filerProxy" { + fileUrl = fmt.Sprintf("http://%s/?proxyChunkId=%s", wfs.getCurrentFiler(), fileId) + } + uploadOption := &operation.UploadOption{ + UploadUrl: fileUrl, + Filename: filename, + Cipher: wfs.option.Cipher, + IsInputCompressed: false, + MimeType: "", + PairMap: nil, + Jwt: auth, + } + uploadResult, err, data := operation.Upload(reader, uploadOption) + if err != nil { + glog.V(0).Infof("upload data %v to %s: %v", filename, fileUrl, err) + return nil, "", "", fmt.Errorf("upload data: %v", err) + } + if uploadResult.Error != "" { + glog.V(0).Infof("upload failure %v to %s: %v", filename, fileUrl, err) + return nil, "", "", fmt.Errorf("upload result: %v", uploadResult.Error) + } + + if offset == 0 { + wfs.chunkCache.SetChunk(fileId, data) + } + + chunk = uploadResult.ToPbFileChunk(fileId, offset) + return chunk, collection, replication, nil + } +} diff --git a/weed/mount/weedfs_xattr.go b/weed/mount/weedfs_xattr.go new file mode 100644 index 000000000..389e86148 --- /dev/null +++ b/weed/mount/weedfs_xattr.go @@ -0,0 +1,168 @@ +package mount + +import ( + "github.com/hanwen/go-fuse/v2/fuse" + sys "golang.org/x/sys/unix" + "runtime" + "strings" + "syscall" +) + +const ( + // https://man7.org/linux/man-pages/man7/xattr.7.html#:~:text=The%20VFS%20imposes%20limitations%20that,in%20listxattr(2)). + MAX_XATTR_NAME_SIZE = 255 + MAX_XATTR_VALUE_SIZE = 65536 + XATTR_PREFIX = "xattr-" // same as filer +) + +// GetXAttr reads an extended attribute, and should return the +// number of bytes. If the buffer is too small, return ERANGE, +// with the required buffer size. +func (wfs *WFS) GetXAttr(cancel <-chan struct{}, header *fuse.InHeader, attr string, dest []byte) (size uint32, code fuse.Status) { + + //validate attr name + if len(attr) > MAX_XATTR_NAME_SIZE { + if runtime.GOOS == "darwin" { + return 0, fuse.EPERM + } else { + return 0, fuse.ERANGE + } + } + if len(attr) == 0 { + return 0, fuse.EINVAL + } + + _, _, entry, status := wfs.maybeReadEntry(header.NodeId) + if status != fuse.OK { + return 0, status + } + if entry == nil { + return 0, fuse.ENOENT + } + if entry.Extended == nil { + return 0, fuse.ENOATTR + } + data, found := entry.Extended[XATTR_PREFIX+attr] + if !found { + return 0, fuse.ENOATTR + } + if len(dest) < len(data) { + return uint32(len(data)), fuse.ERANGE + } + copy(dest, data) + + return uint32(len(data)), fuse.OK +} + +// SetXAttr writes an extended attribute. +// https://man7.org/linux/man-pages/man2/setxattr.2.html +// By default (i.e., flags is zero), the extended attribute will be +// created if it does not exist, or the value will be replaced if +// the attribute already exists. To modify these semantics, one of +// the following values can be specified in flags: +// +// XATTR_CREATE +// Perform a pure create, which fails if the named attribute +// exists already. +// +// XATTR_REPLACE +// Perform a pure replace operation, which fails if the named +// attribute does not already exist. +func (wfs *WFS) SetXAttr(cancel <-chan struct{}, input *fuse.SetXAttrIn, attr string, data []byte) fuse.Status { + //validate attr name + if len(attr) > MAX_XATTR_NAME_SIZE { + if runtime.GOOS == "darwin" { + return fuse.EPERM + } else { + return fuse.ERANGE + } + } + if len(attr) == 0 { + return fuse.EINVAL + } + //validate attr value + if len(data) > MAX_XATTR_VALUE_SIZE { + if runtime.GOOS == "darwin" { + return fuse.Status(syscall.E2BIG) + } else { + return fuse.ERANGE + } + } + + path, _, entry, status := wfs.maybeReadEntry(input.NodeId) + if status != fuse.OK { + return status + } + if entry.Extended == nil { + entry.Extended = make(map[string][]byte) + } + oldData, _ := entry.Extended[XATTR_PREFIX+attr] + switch input.Flags { + case sys.XATTR_CREATE: + if len(oldData) > 0 { + break + } + fallthrough + case sys.XATTR_REPLACE: + fallthrough + default: + entry.Extended[XATTR_PREFIX+attr] = data + } + + return wfs.saveEntry(path, entry) + +} + +// ListXAttr lists extended attributes as '\0' delimited byte +// slice, and return the number of bytes. If the buffer is too +// small, return ERANGE, with the required buffer size. +func (wfs *WFS) ListXAttr(cancel <-chan struct{}, header *fuse.InHeader, dest []byte) (n uint32, code fuse.Status) { + _, _, entry, status := wfs.maybeReadEntry(header.NodeId) + if status != fuse.OK { + return 0, status + } + if entry == nil { + return 0, fuse.ENOENT + } + if entry.Extended == nil { + return 0, fuse.ENOATTR + } + + var data []byte + for k := range entry.Extended { + if strings.HasPrefix(k, XATTR_PREFIX) { + data = append(data, k[len(XATTR_PREFIX):]...) + data = append(data, 0) + } + } + if len(dest) < len(data) { + return uint32(len(data)), fuse.ERANGE + } + + copy(dest, data) + + return uint32(len(data)), fuse.OK +} + +// RemoveXAttr removes an extended attribute. +func (wfs *WFS) RemoveXAttr(cancel <-chan struct{}, header *fuse.InHeader, attr string) fuse.Status { + if len(attr) == 0 { + return fuse.EINVAL + } + path, _, entry, status := wfs.maybeReadEntry(header.NodeId) + if status != fuse.OK { + return status + } + if entry.Extended == nil { + return fuse.ENOATTR + } + _, found := entry.Extended[XATTR_PREFIX+attr] + + if !found { + return fuse.ENOATTR + } + + delete(entry.Extended, XATTR_PREFIX+attr) + + return wfs.saveEntry(path, entry) +} diff --git a/weed/mount/wfs_filer_client.go b/weed/mount/wfs_filer_client.go new file mode 100644 index 000000000..e8feb8342 --- /dev/null +++ b/weed/mount/wfs_filer_client.go @@ -0,0 +1,51 @@ +package mount + +import ( + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/util" + "google.golang.org/grpc" + + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" +) + +var _ = filer_pb.FilerClient(&WFS{}) + +func (wfs *WFS) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) (err error) { + + return util.Retry("filer grpc", func() error { + + i := wfs.option.filerIndex + n := len(wfs.option.FilerAddresses) + for x := 0; x < n; x++ { + + filerGrpcAddress := wfs.option.FilerAddresses[i].ToGrpcAddress() + err = pb.WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error { + client := filer_pb.NewSeaweedFilerClient(grpcConnection) + return fn(client) + }, filerGrpcAddress, wfs.option.GrpcDialOption) + + if err != nil { + glog.V(0).Infof("WithFilerClient %d %v: %v", x, filerGrpcAddress, err) + } else { + wfs.option.filerIndex = i + return nil + } + + i++ + if i >= n { + i = 0 + } + + } + return err + }) + +} + +func (wfs *WFS) AdjustedUrl(location *filer_pb.Location) string { + if wfs.option.VolumeServerAccess == "publicUrl" { + return location.PublicUrl + } + return location.Url +} diff --git a/weed/mount/wfs_save.go b/weed/mount/wfs_save.go new file mode 100644 index 000000000..0cac30453 --- /dev/null +++ b/weed/mount/wfs_save.go @@ -0,0 +1,67 @@ +package mount + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/hanwen/go-fuse/v2/fuse" + "syscall" +) + +func (wfs *WFS) saveEntry(path util.FullPath, entry *filer_pb.Entry) (code fuse.Status) { + + parentDir, _ := path.DirAndName() + + err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + + wfs.mapPbIdFromLocalToFiler(entry) + defer wfs.mapPbIdFromFilerToLocal(entry) + + request := &filer_pb.UpdateEntryRequest{ + Directory: parentDir, + Entry: entry, + Signatures: []int32{wfs.signature}, + } + + glog.V(1).Infof("save entry: %v", request) + _, err := client.UpdateEntry(context.Background(), request) + if err != nil { + return fmt.Errorf("UpdateEntry dir %s: %v", path, err) + } + + if err := wfs.metaCache.UpdateEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil { + return fmt.Errorf("UpdateEntry dir %s: %v", path, err) + } + + return nil + }) + if err != nil { + glog.Errorf("saveEntry %s: %v", path, err) + return fuse.EIO + } + + return fuse.OK +} + +func (wfs *WFS) mapPbIdFromFilerToLocal(entry *filer_pb.Entry) { + if entry.Attributes == nil { + return + } + entry.Attributes.Uid, entry.Attributes.Gid = wfs.option.UidGidMapper.FilerToLocal(entry.Attributes.Uid, entry.Attributes.Gid) +} +func (wfs *WFS) mapPbIdFromLocalToFiler(entry *filer_pb.Entry) { + if entry.Attributes == nil { + return + } + entry.Attributes.Uid, entry.Attributes.Gid = wfs.option.UidGidMapper.LocalToFiler(entry.Attributes.Uid, entry.Attributes.Gid) +} + +func checkName(name string) fuse.Status { + if len(name) >= 256 { + return fuse.Status(syscall.ENAMETOOLONG) + } + return fuse.OK +} diff --git a/weed/replication/sink/filersink/filer_sink.go b/weed/replication/sink/filersink/filer_sink.go index c48ab2368..345c7f13b 100644 --- a/weed/replication/sink/filersink/filer_sink.go +++ b/weed/replication/sink/filersink/filer_sink.go @@ -199,7 +199,7 @@ func (fs *FilerSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParent // delete the chunks that are deleted from the source if deleteIncludeChunks { // remove the deleted chunks. Actual data deletion happens in filer UpdateEntry FindUnusedFileChunks - existingEntry.Chunks = filer.DoMinusChunks(existingEntry.Chunks, deletedChunks) + existingEntry.Chunks = filer.DoMinusChunksBySourceFileId(existingEntry.Chunks, deletedChunks) } // replicate the chunks that are new in the source diff --git a/weed/s3api/s3api_object_multipart_handlers.go b/weed/s3api/s3api_object_multipart_handlers.go index 8cbaf9393..99c280e13 100644 --- a/weed/s3api/s3api_object_multipart_handlers.go +++ b/weed/s3api/s3api_object_multipart_handlers.go @@ -163,13 +163,13 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re UploadId: aws.String(uploadID), }) - glog.V(2).Infof("ListObjectPartsHandler %s count=%d", string(s3err.EncodeXMLResponse(response)), len(response.Part)) - if errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) return } + glog.V(2).Infof("ListObjectPartsHandler %s count=%d", string(s3err.EncodeXMLResponse(response)), len(response.Part)) + writeSuccessResponseXML(w, r, response) } diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 56aee18be..8037b1d94 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -21,6 +21,64 @@ import ( "github.com/chrislusf/seaweedfs/weed/util" ) + +// Validates the preconditions. Returns true if GET/HEAD operation should not proceed. +// Preconditions supported are: +// If-Modified-Since +// If-Unmodified-Since +// If-Match +// If-None-Match +func checkPreconditions(w http.ResponseWriter, r *http.Request, entry *filer.Entry) bool { + + etag := filer.ETagEntry(entry) + /// When more than one conditional request header field is present in a + /// request, the order in which the fields are evaluated becomes + /// important. In practice, the fields defined in this document are + /// consistently implemented in a single, logical order, since "lost + /// update" preconditions have more strict requirements than cache + /// validation, a validated cache is more efficient than a partial + /// response, and entity tags are presumed to be more accurate than date + /// validators. https://tools.ietf.org/html/rfc7232#section-5 + if entry.Attr.Mtime.IsZero() { + return false + } + w.Header().Set("Last-Modified", entry.Attr.Mtime.UTC().Format(http.TimeFormat)) + + ifMatchETagHeader := r.Header.Get("If-Match") + ifUnmodifiedSinceHeader := r.Header.Get("If-Unmodified-Since") + if ifMatchETagHeader != "" { + if util.CanonicalizeETag(etag) != util.CanonicalizeETag(ifMatchETagHeader) { + w.WriteHeader(http.StatusPreconditionFailed) + return true + } + } else if ifUnmodifiedSinceHeader != "" { + if t, parseError := time.Parse(http.TimeFormat, ifUnmodifiedSinceHeader); parseError == nil { + if t.Before(entry.Attr.Mtime) { + w.WriteHeader(http.StatusPreconditionFailed) + return true + } + } + } + + ifNoneMatchETagHeader := r.Header.Get("If-None-Match") + ifModifiedSinceHeader := r.Header.Get("If-Modified-Since") + if ifNoneMatchETagHeader != "" { + if util.CanonicalizeETag(etag) == util.CanonicalizeETag(ifNoneMatchETagHeader) { + w.WriteHeader(http.StatusNotModified) + return true + } + } else if ifModifiedSinceHeader != "" { + if t, parseError := time.Parse(http.TimeFormat, ifModifiedSinceHeader); parseError == nil { + if t.After(entry.Attr.Mtime) { + w.WriteHeader(http.StatusNotModified) + return true + } + } + } + + return false +} + func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) { path := r.URL.Path @@ -61,10 +119,8 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) return } - // set etag etag := filer.ETagEntry(entry) - if ifm := r.Header.Get("If-Match"); ifm != "" && (ifm != "\""+etag+"\"" && ifm != etag) { - w.WriteHeader(http.StatusPreconditionFailed) + if checkPreconditions(w, r, entry) { return } @@ -81,19 +137,6 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) w.Header().Set("Content-Type", mimeType) } - // if modified since - if !entry.Attr.Mtime.IsZero() { - w.Header().Set("Last-Modified", entry.Attr.Mtime.UTC().Format(http.TimeFormat)) - if r.Header.Get("If-Modified-Since") != "" { - if t, parseError := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); parseError == nil { - if !t.Before(entry.Attr.Mtime) { - w.WriteHeader(http.StatusNotModified) - return - } - } - } - } - // print out the header from extended properties for k, v := range entry.Extended { if !strings.HasPrefix(k, "xattr-") { @@ -123,10 +166,6 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) w.Header().Set(xhttp.AmzTagCount, strconv.Itoa(tagCount)) } - if inm := r.Header.Get("If-None-Match"); inm == "\""+etag+"\"" { - w.WriteHeader(http.StatusNotModified) - return - } setEtag(w, etag) filename := entry.Name() diff --git a/weed/server/raft_server.go b/weed/server/raft_server.go index 568bfc7b5..91dd185c8 100644 --- a/weed/server/raft_server.go +++ b/weed/server/raft_server.go @@ -19,6 +19,17 @@ import ( "github.com/chrislusf/seaweedfs/weed/topology" ) +type RaftServerOption struct { + GrpcDialOption grpc.DialOption + Peers []pb.ServerAddress + ServerAddr pb.ServerAddress + DataDir string + Topo *topology.Topology + RaftResumeState bool + HeartbeatInterval time.Duration + ElectionTimeout time.Duration +} + type RaftServer struct { peers []pb.ServerAddress // initial peers to join with raftServer raft.Server @@ -52,12 +63,12 @@ func (s StateMachine) Recovery(data []byte) error { return nil } -func NewRaftServer(grpcDialOption grpc.DialOption, peers []pb.ServerAddress, serverAddr pb.ServerAddress, dataDir string, topo *topology.Topology, raftResumeState bool) (*RaftServer, error) { +func NewRaftServer(option *RaftServerOption) (*RaftServer, error) { s := &RaftServer{ - peers: peers, - serverAddr: serverAddr, - dataDir: dataDir, - topo: topo, + peers: option.Peers, + serverAddr: option.ServerAddr, + dataDir: option.DataDir, + topo: option.Topo, } if glog.V(4) { @@ -67,10 +78,10 @@ func NewRaftServer(grpcDialOption grpc.DialOption, peers []pb.ServerAddress, ser raft.RegisterCommand(&topology.MaxVolumeIdCommand{}) var err error - transporter := raft.NewGrpcTransporter(grpcDialOption) - glog.V(0).Infof("Starting RaftServer with %v", serverAddr) + transporter := raft.NewGrpcTransporter(option.GrpcDialOption) + glog.V(0).Infof("Starting RaftServer with %v", option.ServerAddr) - if !raftResumeState { + if !option.RaftResumeState { // always clear previous metadata os.RemoveAll(path.Join(s.dataDir, "conf")) os.RemoveAll(path.Join(s.dataDir, "log")) @@ -80,14 +91,15 @@ func NewRaftServer(grpcDialOption grpc.DialOption, peers []pb.ServerAddress, ser return nil, err } - stateMachine := StateMachine{topo: topo} - s.raftServer, err = raft.NewServer(string(s.serverAddr), s.dataDir, transporter, stateMachine, topo, "") + stateMachine := StateMachine{topo: option.Topo} + s.raftServer, err = raft.NewServer(string(s.serverAddr), s.dataDir, transporter, stateMachine, option.Topo, "") if err != nil { glog.V(0).Infoln(err) return nil, err } - s.raftServer.SetHeartbeatInterval(time.Duration(300+rand.Intn(150)) * time.Millisecond) - s.raftServer.SetElectionTimeout(10 * time.Second) + heartbeatInterval := time.Duration(float64(option.HeartbeatInterval) * (rand.Float64()*0.25 + 1)) + s.raftServer.SetHeartbeatInterval(heartbeatInterval) + s.raftServer.SetElectionTimeout(option.ElectionTimeout) if err := s.raftServer.LoadSnapshot(); err != nil { return nil, err } @@ -123,7 +135,7 @@ func NewRaftServer(grpcDialOption grpc.DialOption, peers []pb.ServerAddress, ser s.GrpcServer = raft.NewGrpcServer(s.raftServer) - if s.raftServer.IsLogEmpty() && isTheFirstOne(serverAddr, s.peers) { + if s.raftServer.IsLogEmpty() && isTheFirstOne(option.ServerAddr, s.peers) { // Initialize the server by joining itself. // s.DoJoinCommand() } diff --git a/weed/shell/command_collection_list.go b/weed/shell/command_collection_list.go index bc9f115bd..55fd6d9b9 100644 --- a/weed/shell/command_collection_list.go +++ b/weed/shell/command_collection_list.go @@ -38,7 +38,7 @@ func (c *commandCollectionList) Do(args []string, commandEnv *CommandEnv, writer return err } - topologyInfo, _, err := collectTopologyInfo(commandEnv) + topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) if err != nil { return err } diff --git a/weed/shell/command_ec_common.go b/weed/shell/command_ec_common.go index 765c0ad89..b3bd0ce5d 100644 --- a/weed/shell/command_ec_common.go +++ b/weed/shell/command_ec_common.go @@ -172,7 +172,7 @@ func countFreeShardSlots(dn *master_pb.DataNodeInfo, diskType types.DiskType) (c if diskInfo == nil { return 0 } - return int(diskInfo.MaxVolumeCount-diskInfo.ActiveVolumeCount)*erasure_coding.DataShardsCount - countShards(diskInfo.EcShardInfos) + return int(diskInfo.MaxVolumeCount-diskInfo.VolumeCount)*erasure_coding.DataShardsCount - countShards(diskInfo.EcShardInfos) } type RackId string @@ -206,7 +206,7 @@ func collectEcNodes(commandEnv *CommandEnv, selectedDataCenter string) (ecNodes // list all possible locations // collect topology information - topologyInfo, _, err := collectTopologyInfo(commandEnv) + topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) if err != nil { return } diff --git a/weed/shell/command_ec_decode.go b/weed/shell/command_ec_decode.go index 288fa4945..de22990fa 100644 --- a/weed/shell/command_ec_decode.go +++ b/weed/shell/command_ec_decode.go @@ -8,6 +8,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/storage/types" "io" + "time" "google.golang.org/grpc" @@ -53,7 +54,7 @@ func (c *commandEcDecode) Do(args []string, commandEnv *CommandEnv, writer io.Wr vid := needle.VolumeId(*volumeId) // collect topology information - topologyInfo, _, err := collectTopologyInfo(commandEnv) + topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) if err != nil { return err } @@ -233,7 +234,11 @@ func lookupVolumeIds(commandEnv *CommandEnv, volumeIds []string) (volumeIdLocati return resp.VolumeIdLocations, nil } -func collectTopologyInfo(commandEnv *CommandEnv) (topoInfo *master_pb.TopologyInfo, volumeSizeLimitMb uint64, err error) { +func collectTopologyInfo(commandEnv *CommandEnv, delayBeforeCollecting time.Duration) (topoInfo *master_pb.TopologyInfo, volumeSizeLimitMb uint64, err error) { + + if delayBeforeCollecting > 0 { + time.Sleep(delayBeforeCollecting) + } var resp *master_pb.VolumeListResponse err = commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error { diff --git a/weed/shell/command_ec_encode.go b/weed/shell/command_ec_encode.go index fcdee264e..9ae9b049c 100644 --- a/weed/shell/command_ec_encode.go +++ b/weed/shell/command_ec_encode.go @@ -267,7 +267,7 @@ func balancedEcDistribution(servers []*EcNode) (allocated [][]uint32) { func collectVolumeIdsForEcEncode(commandEnv *CommandEnv, selectedCollection string, fullPercentage float64, quietPeriod time.Duration) (vids []needle.VolumeId, err error) { // collect topology information - topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv) + topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv, 0) if err != nil { return } diff --git a/weed/shell/command_ec_encode_test.go b/weed/shell/command_ec_encode_test.go new file mode 100644 index 000000000..d5e341e5b --- /dev/null +++ b/weed/shell/command_ec_encode_test.go @@ -0,0 +1,30 @@ +package shell + +import ( + "fmt" + "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding" + "testing" +) + +func TestEcDistribution(t *testing.T) { + + topologyInfo := parseOutput(topoData) + + // find out all volume servers with one slot left. + ecNodes, totalFreeEcSlots := collectEcVolumeServersByDc(topologyInfo, "") + + sortEcNodesByFreeslotsDecending(ecNodes) + + if totalFreeEcSlots < erasure_coding.TotalShardsCount { + println("not enough free ec shard slots", totalFreeEcSlots) + } + allocatedDataNodes := ecNodes + if len(allocatedDataNodes) > erasure_coding.TotalShardsCount { + allocatedDataNodes = allocatedDataNodes[:erasure_coding.TotalShardsCount] + } + + for _, dn := range allocatedDataNodes { + fmt.Printf("info %+v %+v\n", dn.info, dn) + } + +} diff --git a/weed/shell/command_fs_cat.go b/weed/shell/command_fs_cat.go index 17e9c6550..16be25ee5 100644 --- a/weed/shell/command_fs_cat.go +++ b/weed/shell/command_fs_cat.go @@ -2,12 +2,10 @@ package shell import ( "fmt" - "io" - "math" - "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" + "io" ) func init() { @@ -57,7 +55,7 @@ func (c *commandFsCat) Do(args []string, commandEnv *CommandEnv, writer io.Write return err } - return filer.StreamContent(commandEnv.MasterClient, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64) + return filer.StreamContent(commandEnv.MasterClient, writer, respLookupEntry.Entry.Chunks, 0, int64(filer.FileSize(respLookupEntry.Entry))) }) diff --git a/weed/shell/command_s3_bucket_list.go b/weed/shell/command_s3_bucket_list.go index 37bb0ec20..65297e239 100644 --- a/weed/shell/command_s3_bucket_list.go +++ b/weed/shell/command_s3_bucket_list.go @@ -35,7 +35,7 @@ func (c *commandS3BucketList) Do(args []string, commandEnv *CommandEnv, writer i } // collect collection information - topologyInfo, _, err := collectTopologyInfo(commandEnv) + topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) if err != nil { return err } diff --git a/weed/shell/command_s3_bucket_quota_check.go b/weed/shell/command_s3_bucket_quota_check.go index 742877c1c..066ef6909 100644 --- a/weed/shell/command_s3_bucket_quota_check.go +++ b/weed/shell/command_s3_bucket_quota_check.go @@ -38,7 +38,7 @@ func (c *commandS3BucketQuotaEnforce) Do(args []string, commandEnv *CommandEnv, } // collect collection information - topologyInfo, _, err := collectTopologyInfo(commandEnv) + topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) if err != nil { return err } diff --git a/weed/shell/command_volume_balance.go b/weed/shell/command_volume_balance.go index 4c55de5fb..7a983de1a 100644 --- a/weed/shell/command_volume_balance.go +++ b/weed/shell/command_volume_balance.go @@ -76,7 +76,7 @@ func (c *commandVolumeBalance) Do(args []string, commandEnv *CommandEnv, writer } // collect topology information - topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv) + topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv, 15*time.Second) if err != nil { return err } diff --git a/weed/shell/command_volume_check_disk.go b/weed/shell/command_volume_check_disk.go index a107574a8..54edd53dd 100644 --- a/weed/shell/command_volume_check_disk.go +++ b/weed/shell/command_volume_check_disk.go @@ -58,7 +58,7 @@ func (c *commandVolumeCheckDisk) Do(args []string, commandEnv *CommandEnv, write c.env = commandEnv // collect topology information - topologyInfo, _, err := collectTopologyInfo(commandEnv) + topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) if err != nil { return err } diff --git a/weed/shell/command_volume_configure_replication.go b/weed/shell/command_volume_configure_replication.go index 5c4c10146..610986489 100644 --- a/weed/shell/command_volume_configure_replication.go +++ b/weed/shell/command_volume_configure_replication.go @@ -59,7 +59,7 @@ func (c *commandVolumeConfigureReplication) Do(args []string, commandEnv *Comman } // collect topology information - topologyInfo, _, err := collectTopologyInfo(commandEnv) + topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) if err != nil { return err } diff --git a/weed/shell/command_volume_delete_empty.go b/weed/shell/command_volume_delete_empty.go index c98693147..2d842e80a 100644 --- a/weed/shell/command_volume_delete_empty.go +++ b/weed/shell/command_volume_delete_empty.go @@ -45,7 +45,7 @@ func (c *commandVolumeDeleteEmpty) Do(args []string, commandEnv *CommandEnv, wri } // collect topology information - topologyInfo, _, err := collectTopologyInfo(commandEnv) + topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) if err != nil { return err } diff --git a/weed/shell/command_volume_fix_replication.go b/weed/shell/command_volume_fix_replication.go index 43bf4d0f8..78285d8a5 100644 --- a/weed/shell/command_volume_fix_replication.go +++ b/weed/shell/command_volume_fix_replication.go @@ -75,7 +75,7 @@ func (c *commandVolumeFixReplication) Do(args []string, commandEnv *CommandEnv, fixedVolumeReplicas := map[string]int{} // collect topology information - topologyInfo, _, err := collectTopologyInfo(commandEnv) + topologyInfo, _, err := collectTopologyInfo(commandEnv, 15*time.Second) if err != nil { return err } diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index e73f60a2b..28f2d6753 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -512,7 +512,7 @@ func (c *commandVolumeFsck) collectVolumeIds(commandEnv *CommandEnv, verbose boo volumeIdToServer = make(map[uint32]VInfo) // collect topology information - topologyInfo, _, err := collectTopologyInfo(commandEnv) + topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) if err != nil { return } diff --git a/weed/shell/command_volume_list.go b/weed/shell/command_volume_list.go index 531f7f675..4c0429ecb 100644 --- a/weed/shell/command_volume_list.go +++ b/weed/shell/command_volume_list.go @@ -39,7 +39,7 @@ func (c *commandVolumeList) Do(args []string, commandEnv *CommandEnv, writer io. } // collect topology information - topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv) + topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv, 0) if err != nil { return err } diff --git a/weed/shell/command_volume_list_test.go b/weed/shell/command_volume_list_test.go index 379cf4943..0bcbb1713 100644 --- a/weed/shell/command_volume_list_test.go +++ b/weed/shell/command_volume_list_test.go @@ -1,6 +1,7 @@ package shell import ( + _ "embed" "github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" @@ -64,11 +65,14 @@ func parseOutput(output string) *master_pb.TopologyInfo { case "Disk": if disk == nil { diskType := parts[1][:strings.Index(parts[1], "(")] + volumeCountStr := parts[1][strings.Index(parts[1], ":")+1 : strings.Index(parts[1], "/")] maxVolumeCountStr := parts[1][strings.Index(parts[1], "/")+1:] maxVolumeCount, _ := strconv.Atoi(maxVolumeCountStr) + volumeCount, _ := strconv.Atoi(volumeCountStr) disk = &master_pb.DiskInfo{ Type: diskType, MaxVolumeCount: int64(maxVolumeCount), + VolumeCount: int64(volumeCount), } dn.DiskInfos[types.ToDiskType(diskType).String()] = disk } else { @@ -85,809 +89,5 @@ func parseOutput(output string) *master_pb.TopologyInfo { return topo } -const topoData = ` -Topology volumeSizeLimit:1024 MB hdd(volume:760/7280 active:760 free:6520 remote:0) - DataCenter dc1 hdd(volume:0/0 active:0 free:0 remote:0) - Rack DefaultRack hdd(volume:0/0 active:0 free:0 remote:0) - Rack DefaultRack total size:0 file_count:0 - DataCenter dc1 total size:0 file_count:0 - DataCenter dc2 hdd(volume:86/430 active:86 free:344 remote:0) - Rack rack1 hdd(volume:50/240 active:50 free:190 remote:0) - DataNode 192.168.1.4:8080 hdd(volume:50/240 active:50 free:190 remote:0) - Disk hdd(volume:50/240 active:50 free:190 remote:0) - volume id:15 size:1115965064 collection:"collection0" file_count:83 replica_placement:100 version:3 modified_at_second:1609923671 - volume id:21 size:1097631536 collection:"collection0" file_count:82 delete_count:7 deleted_byte_count:68975485 replica_placement:100 version:3 modified_at_second:1609929578 - volume id:22 size:1086828272 collection:"collection0" file_count:75 replica_placement:100 version:3 modified_at_second:1609930001 - volume id:23 size:1076380216 collection:"collection0" file_count:68 replica_placement:100 version:3 modified_at_second:1609930434 - volume id:24 size:1074139776 collection:"collection0" file_count:90 replica_placement:100 version:3 modified_at_second:1609930909 - volume id:25 size:690757512 collection:"collection0" file_count:38 replica_placement:100 version:3 modified_at_second:1611144216 - volume id:27 size:298886792 file_count:1608 replica_placement:100 version:3 modified_at_second:1615632482 - volume id:28 size:308919192 file_count:1591 delete_count:1 deleted_byte_count:125280 replica_placement:100 version:3 modified_at_second:1615631762 - volume id:29 size:281582680 file_count:1537 replica_placement:100 version:3 modified_at_second:1615629422 - volume id:30 size:289466144 file_count:1566 delete_count:1 deleted_byte_count:124972 replica_placement:100 version:3 modified_at_second:1615632422 - volume id:31 size:273363256 file_count:1498 replica_placement:100 version:3 modified_at_second:1615631642 - volume id:33 size:1130226400 collection:"collection1" file_count:7322 delete_count:172 deleted_byte_count:45199399 replica_placement:100 version:3 modified_at_second:1615618789 - volume id:38 size:1075545744 collection:"collection1" file_count:13324 delete_count:100 deleted_byte_count:25223906 replica_placement:100 version:3 modified_at_second:1615569830 - volume id:51 size:1076796120 collection:"collection1" file_count:10550 delete_count:39 deleted_byte_count:12723654 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615547786 - volume id:52 size:1083529728 collection:"collection1" file_count:10128 delete_count:32 deleted_byte_count:10608391 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615599195 - volume id:54 size:1045022344 collection:"collection1" file_count:9408 delete_count:30 deleted_byte_count:15132106 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630812 - volume id:63 size:956941112 collection:"collection1" file_count:8271 delete_count:32 deleted_byte_count:15876189 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632036 - volume id:69 size:869213648 collection:"collection1" file_count:7293 delete_count:102 deleted_byte_count:30643207 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630534 - volume id:74 size:957046128 collection:"collection1" file_count:6982 delete_count:258 deleted_byte_count:73054259 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631460 - volume id:80 size:827912928 collection:"collection1" file_count:6914 delete_count:17 deleted_byte_count:5689635 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631157 - volume id:84 size:873121856 collection:"collection1" file_count:8200 delete_count:13 deleted_byte_count:3131676 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631161 - volume id:85 size:1023869320 collection:"collection1" file_count:7788 delete_count:234 deleted_byte_count:78037967 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631723 - volume id:97 size:1053112992 collection:"collection1" file_count:6789 delete_count:50 deleted_byte_count:38894001 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631193 - volume id:98 size:1077836440 collection:"collection1" file_count:7605 delete_count:202 deleted_byte_count:73180379 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615523691 - volume id:105 size:1073996824 collection:"collection1" file_count:6872 delete_count:20 deleted_byte_count:14482293 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615499757 - volume id:106 size:1075458664 collection:"collection1" file_count:7182 delete_count:307 deleted_byte_count:69349053 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615598137 - volume id:112 size:1076392512 collection:"collection1" file_count:8291 delete_count:156 deleted_byte_count:74120183 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615569823 - volume id:116 size:1074489504 collection:"collection1" file_count:9981 delete_count:174 deleted_byte_count:53998777 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615611565 - volume id:119 size:1075940104 collection:"collection1" file_count:9003 delete_count:12 deleted_byte_count:9128155 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615573878 - volume id:128 size:1074874632 collection:"collection1" file_count:9821 delete_count:148 deleted_byte_count:43633334 replica_placement:100 version:3 modified_at_second:1615602670 - volume id:133 size:1075952760 collection:"collection1" file_count:9538 delete_count:74 deleted_byte_count:19558008 replica_placement:100 version:3 modified_at_second:1615584779 - volume id:136 size:1074433552 collection:"collection1" file_count:9593 delete_count:72 deleted_byte_count:26912512 replica_placement:100 version:3 modified_at_second:1615376036 - volume id:138 size:1074465744 collection:"collection1" file_count:10120 delete_count:55 deleted_byte_count:15875438 replica_placement:100 version:3 modified_at_second:1615572231 - volume id:140 size:1076203744 collection:"collection1" file_count:11219 delete_count:57 deleted_byte_count:19864498 replica_placement:100 version:3 modified_at_second:1615571947 - volume id:144 size:1074549720 collection:"collection1" file_count:8780 delete_count:50 deleted_byte_count:52475146 replica_placement:100 version:3 modified_at_second:1615573451 - volume id:161 size:1077397192 collection:"collection1" file_count:9988 delete_count:28 deleted_byte_count:12509164 replica_placement:100 version:3 modified_at_second:1615631452 - volume id:173 size:1074154704 collection:"collection1" file_count:30884 delete_count:34 deleted_byte_count:2578509 replica_placement:100 version:3 modified_at_second:1615591904 - volume id:174 size:1073824232 collection:"collection1" file_count:30689 delete_count:36 deleted_byte_count:2160116 replica_placement:100 version:3 modified_at_second:1615598914 - volume id:197 size:1075423240 collection:"collection1" file_count:16473 delete_count:15 deleted_byte_count:12552442 replica_placement:100 version:3 modified_at_second:1615485254 - volume id:219 size:1092298904 collection:"collection1" file_count:3193 delete_count:17 deleted_byte_count:2047576 replica_placement:100 version:3 modified_at_second:1615579316 - volume id:263 size:1077167352 collection:"collection2" file_count:20227 delete_count:4 deleted_byte_count:97887 replica_placement:100 version:3 modified_at_second:1614871567 - volume id:272 size:1076146040 collection:"collection2" file_count:21034 delete_count:2 deleted_byte_count:216564 replica_placement:100 version:3 modified_at_second:1614884139 - volume id:291 size:1076256760 collection:"collection2" file_count:28301 delete_count:5 deleted_byte_count:116027 replica_placement:100 version:3 modified_at_second:1614904924 - volume id:299 size:1075147824 collection:"collection2" file_count:22927 delete_count:4 deleted_byte_count:345569 replica_placement:100 version:3 modified_at_second:1614918454 - volume id:301 size:1074655600 collection:"collection2" file_count:22543 delete_count:6 deleted_byte_count:136968 replica_placement:100 version:3 modified_at_second:1614918378 - volume id:302 size:1077559792 collection:"collection2" file_count:23124 delete_count:7 deleted_byte_count:293111 replica_placement:100 version:3 modified_at_second:1614925500 - volume id:339 size:1078402392 collection:"collection2" file_count:22309 replica_placement:100 version:3 modified_at_second:1614969996 - volume id:345 size:1074560760 collection:"collection2" file_count:22117 delete_count:2 deleted_byte_count:373286 replica_placement:100 version:3 modified_at_second:1614977458 - volume id:355 size:1075239792 collection:"collection2" file_count:22244 delete_count:1 deleted_byte_count:23282 replica_placement:100 version:3 modified_at_second:1614992157 - volume id:373 size:1080928000 collection:"collection2" file_count:22617 delete_count:4 deleted_byte_count:91849 replica_placement:100 version:3 modified_at_second:1615016877 - Disk hdd total size:48630015544 file_count:537880 deleted_file:2580 deleted_bytes:929560253 - DataNode 192.168.1.4:8080 total size:48630015544 file_count:537880 deleted_file:2580 deleted_bytes:929560253 - Rack rack1 total size:48630015544 file_count:537880 deleted_file:2580 deleted_bytes:929560253 - Rack rack2 hdd(volume:36/190 active:36 free:154 remote:0) - DataNode 192.168.1.2:8080 hdd(volume:36/190 active:36 free:154 remote:0) - Disk hdd(volume:36/190 active:36 free:154 remote:0) - volume id:2 size:289228560 file_count:1640 delete_count:4 deleted_byte_count:480564 replica_placement:100 version:3 compact_revision:6 modified_at_second:1615630622 - volume id:3 size:308743136 file_count:1638 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632242 - volume id:4 size:285986968 file_count:1641 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632302 - volume id:6 size:302411024 file_count:1604 delete_count:2 deleted_byte_count:274587 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631402 - volume id:7 size:1924728 collection:"collection4" file_count:15 replica_placement:100 version:3 modified_at_second:1609331040 - volume id:9 size:77337416 collection:"collection3" file_count:58 replica_placement:100 version:3 ttl:772 modified_at_second:1615513762 - volume id:10 size:1212784656 collection:"collection0" file_count:58 replica_placement:100 version:3 modified_at_second:1609814550 - volume id:12 size:1110923848 collection:"collection0" file_count:45 replica_placement:100 version:3 modified_at_second:1609819732 - volume id:13 size:1184910656 collection:"collection0" file_count:47 replica_placement:100 version:3 modified_at_second:1609827837 - volume id:14 size:1107475720 collection:"collection0" file_count:80 delete_count:3 deleted_byte_count:6870 replica_placement:100 version:3 modified_at_second:1612956980 - volume id:16 size:1113666104 collection:"collection0" file_count:73 delete_count:5 deleted_byte_count:6318 replica_placement:100 version:3 modified_at_second:1612957007 - volume id:17 size:1095115800 collection:"collection0" file_count:83 delete_count:3 deleted_byte_count:7099 replica_placement:100 version:3 modified_at_second:1612957000 - volume id:21 size:1097631664 collection:"collection0" file_count:82 delete_count:11 deleted_byte_count:68985100 replica_placement:100 version:3 modified_at_second:1612957007 - volume id:56 size:1001897616 collection:"collection1" file_count:8762 delete_count:37 deleted_byte_count:65375405 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632014 - volume id:81 size:880693104 collection:"collection1" file_count:7481 delete_count:236 deleted_byte_count:80386421 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631396 - volume id:104 size:1076383624 collection:"collection1" file_count:7663 delete_count:184 deleted_byte_count:100728071 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615602658 - volume id:107 size:1073811840 collection:"collection1" file_count:7436 delete_count:168 deleted_byte_count:57747484 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615293569 - volume id:113 size:1076709184 collection:"collection1" file_count:9355 delete_count:177 deleted_byte_count:59796765 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615569822 - volume id:139 size:1074163936 collection:"collection1" file_count:9315 delete_count:42 deleted_byte_count:10630966 replica_placement:100 version:3 modified_at_second:1615571946 - volume id:151 size:1098659752 collection:"collection1" file_count:10808 delete_count:24 deleted_byte_count:7088102 replica_placement:100 version:3 modified_at_second:1615586389 - volume id:155 size:1075140688 collection:"collection1" file_count:10882 delete_count:32 deleted_byte_count:9076141 replica_placement:100 version:3 modified_at_second:1615606614 - volume id:167 size:1073958176 collection:"collection1" file_count:25229 delete_count:48 deleted_byte_count:25871565 replica_placement:100 version:3 modified_at_second:1615602669 - volume id:177 size:1074120216 collection:"collection1" file_count:22293 delete_count:16 deleted_byte_count:3803952 replica_placement:100 version:3 modified_at_second:1615516892 - volume id:179 size:1074313920 collection:"collection1" file_count:21829 delete_count:24 deleted_byte_count:45552859 replica_placement:100 version:3 modified_at_second:1615580308 - volume id:182 size:1076131280 collection:"collection1" file_count:31987 delete_count:21 deleted_byte_count:1452346 replica_placement:100 version:3 modified_at_second:1615568922 - volume id:215 size:1068268216 collection:"collection1" file_count:2813 delete_count:10 deleted_byte_count:5676795 replica_placement:100 version:3 modified_at_second:1615586386 - volume id:217 size:1075381872 collection:"collection1" file_count:3331 delete_count:14 deleted_byte_count:2009141 replica_placement:100 version:3 modified_at_second:1615401638 - volume id:283 size:1080178944 collection:"collection2" file_count:19462 delete_count:7 deleted_byte_count:660407 replica_placement:100 version:3 modified_at_second:1614896626 - volume id:303 size:1075944504 collection:"collection2" file_count:22541 delete_count:2 deleted_byte_count:13617 replica_placement:100 version:3 modified_at_second:1614925431 - volume id:309 size:1075178624 collection:"collection2" file_count:22692 delete_count:3 deleted_byte_count:171124 replica_placement:100 version:3 modified_at_second:1614931409 - volume id:323 size:1074608200 collection:"collection2" file_count:21605 delete_count:4 deleted_byte_count:172090 replica_placement:100 version:3 modified_at_second:1614950526 - volume id:344 size:1075035448 collection:"collection2" file_count:21765 delete_count:1 deleted_byte_count:24623 replica_placement:100 version:3 modified_at_second:1614977465 - volume id:347 size:1075145496 collection:"collection2" file_count:22178 delete_count:1 deleted_byte_count:79392 replica_placement:100 version:3 modified_at_second:1614984727 - volume id:357 size:1074276208 collection:"collection2" file_count:23137 delete_count:4 deleted_byte_count:188487 replica_placement:100 version:3 modified_at_second:1614998792 - volume id:380 size:1010760456 collection:"collection2" file_count:14921 delete_count:6 deleted_byte_count:65678 replica_placement:100 version:3 modified_at_second:1615632322 - volume id:381 size:939292792 collection:"collection2" file_count:14619 delete_count:2 deleted_byte_count:5119 replica_placement:100 version:3 modified_at_second:1615632324 - Disk hdd total size:33468194376 file_count:369168 deleted_file:1091 deleted_bytes:546337088 - DataNode 192.168.1.2:8080 total size:33468194376 file_count:369168 deleted_file:1091 deleted_bytes:546337088 - Rack rack2 total size:33468194376 file_count:369168 deleted_file:1091 deleted_bytes:546337088 - DataCenter dc2 total size:82098209920 file_count:907048 deleted_file:3671 deleted_bytes:1475897341 - DataCenter dc3 hdd(volume:108/1850 active:108 free:1742 remote:0) - Rack rack3 hdd(volume:108/1850 active:108 free:1742 remote:0) - DataNode 192.168.1.6:8080 hdd(volume:108/1850 active:108 free:1742 remote:0) - Disk hdd(volume:108/1850 active:108 free:1742 remote:0) - volume id:1 size:284685936 file_count:1557 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632062 - volume id:32 size:281390512 file_count:1496 delete_count:6 deleted_byte_count:546403 replica_placement:100 version:3 modified_at_second:1615632362 - volume id:47 size:444599784 collection:"collection1" file_count:709 delete_count:19 deleted_byte_count:11913451 replica_placement:100 version:3 modified_at_second:1615632397 - volume id:49 size:1078775288 collection:"collection1" file_count:9636 delete_count:22 deleted_byte_count:5625976 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630446 - volume id:68 size:898630584 collection:"collection1" file_count:6934 delete_count:95 deleted_byte_count:27460707 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632284 - volume id:88 size:1073767976 collection:"collection1" file_count:14995 delete_count:206 deleted_byte_count:81222360 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615629897 - volume id:202 size:1077533160 collection:"collection1" file_count:2847 delete_count:67 deleted_byte_count:65172985 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615588497 - volume id:203 size:1027316272 collection:"collection1" file_count:3040 delete_count:11 deleted_byte_count:3993230 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631728 - volume id:205 size:1078485304 collection:"collection1" file_count:2869 delete_count:43 deleted_byte_count:18290259 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615579314 - volume id:206 size:1082045848 collection:"collection1" file_count:2979 delete_count:225 deleted_byte_count:88220074 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615564274 - volume id:209 size:1074083592 collection:"collection1" file_count:3238 delete_count:4 deleted_byte_count:1494244 replica_placement:100 version:3 modified_at_second:1615419954 - volume id:211 size:1080610712 collection:"collection1" file_count:3247 delete_count:7 deleted_byte_count:1891456 replica_placement:100 version:3 modified_at_second:1615269124 - volume id:212 size:1078293360 collection:"collection1" file_count:3106 delete_count:6 deleted_byte_count:2085755 replica_placement:100 version:3 modified_at_second:1615586387 - volume id:213 size:1093587976 collection:"collection1" file_count:3681 delete_count:12 deleted_byte_count:3138791 replica_placement:100 version:3 modified_at_second:1615586387 - volume id:214 size:1074486992 collection:"collection1" file_count:3217 delete_count:10 deleted_byte_count:6392871 replica_placement:100 version:3 modified_at_second:1615586383 - volume id:216 size:1080073496 collection:"collection1" file_count:3316 delete_count:4 deleted_byte_count:179819 replica_placement:100 version:3 modified_at_second:1615586387 - volume id:222 size:1106623104 collection:"collection1" file_count:3273 delete_count:11 deleted_byte_count:2114627 replica_placement:100 version:3 modified_at_second:1615586243 - volume id:223 size:1075233064 collection:"collection1" file_count:2966 delete_count:9 deleted_byte_count:744001 replica_placement:100 version:3 modified_at_second:1615586244 - volume id:227 size:1106699896 collection:"collection1" file_count:2827 delete_count:20 deleted_byte_count:5496790 replica_placement:100 version:3 modified_at_second:1615609989 - volume id:229 size:1109855312 collection:"collection1" file_count:2857 delete_count:22 deleted_byte_count:2839883 replica_placement:100 version:3 modified_at_second:1615609988 - volume id:230 size:1080722984 collection:"collection1" file_count:2898 delete_count:15 deleted_byte_count:3929261 replica_placement:100 version:3 modified_at_second:1615610537 - volume id:231 size:1112917696 collection:"collection1" file_count:3151 delete_count:20 deleted_byte_count:2989828 replica_placement:100 version:3 modified_at_second:1615611350 - volume id:233 size:1080526464 collection:"collection1" file_count:3136 delete_count:61 deleted_byte_count:17991717 replica_placement:100 version:3 modified_at_second:1615611352 - volume id:234 size:1073835280 collection:"collection1" file_count:2965 delete_count:41 deleted_byte_count:4960354 replica_placement:100 version:3 modified_at_second:1615611351 - volume id:235 size:1075586104 collection:"collection1" file_count:2767 delete_count:33 deleted_byte_count:3216540 replica_placement:100 version:3 modified_at_second:1615611354 - volume id:237 size:375722792 collection:"collection1" file_count:736 delete_count:16 deleted_byte_count:4464870 replica_placement:100 version:3 modified_at_second:1615631727 - volume id:239 size:426569024 collection:"collection1" file_count:693 delete_count:19 deleted_byte_count:13020783 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630838 - volume id:241 size:380217424 collection:"collection1" file_count:633 delete_count:6 deleted_byte_count:1715768 replica_placement:100 version:3 modified_at_second:1615632006 - volume id:244 size:1080295352 collection:"collection2" file_count:10812 delete_count:1 deleted_byte_count:795 replica_placement:100 version:3 modified_at_second:1614852171 - volume id:245 size:1074597056 collection:"collection2" file_count:10371 delete_count:3 deleted_byte_count:209701 replica_placement:100 version:3 modified_at_second:1614852093 - volume id:246 size:1075998648 collection:"collection2" file_count:10365 delete_count:1 deleted_byte_count:13112 replica_placement:100 version:3 modified_at_second:1614852105 - volume id:248 size:1084301184 collection:"collection2" file_count:11217 delete_count:4 deleted_byte_count:746488 replica_placement:100 version:3 modified_at_second:1614856285 - volume id:249 size:1074819136 collection:"collection2" file_count:10763 delete_count:2 deleted_byte_count:271699 replica_placement:100 version:3 modified_at_second:1614856230 - volume id:251 size:1075684488 collection:"collection2" file_count:10847 replica_placement:100 version:3 modified_at_second:1614856270 - volume id:252 size:1075065208 collection:"collection2" file_count:14622 delete_count:2 deleted_byte_count:5228 replica_placement:100 version:3 modified_at_second:1614861196 - volume id:253 size:1087328816 collection:"collection2" file_count:14920 delete_count:3 deleted_byte_count:522994 replica_placement:100 version:3 modified_at_second:1614861255 - volume id:255 size:1079581640 collection:"collection2" file_count:14877 delete_count:3 deleted_byte_count:101223 replica_placement:100 version:3 modified_at_second:1614861233 - volume id:256 size:1074283592 collection:"collection2" file_count:14157 delete_count:1 deleted_byte_count:18156 replica_placement:100 version:3 modified_at_second:1614861100 - volume id:258 size:1075527216 collection:"collection2" file_count:18421 delete_count:4 deleted_byte_count:267833 replica_placement:100 version:3 modified_at_second:1614866420 - volume id:259 size:1075507776 collection:"collection2" file_count:18079 delete_count:2 deleted_byte_count:71992 replica_placement:100 version:3 modified_at_second:1614866381 - volume id:264 size:1081624192 collection:"collection2" file_count:21151 replica_placement:100 version:3 modified_at_second:1614871629 - volume id:265 size:1076401104 collection:"collection2" file_count:19932 delete_count:2 deleted_byte_count:160823 replica_placement:100 version:3 modified_at_second:1615629130 - volume id:266 size:1075617464 collection:"collection2" file_count:20075 delete_count:1 deleted_byte_count:1039 replica_placement:100 version:3 modified_at_second:1614871526 - volume id:267 size:1075699544 collection:"collection2" file_count:21039 delete_count:3 deleted_byte_count:59956 replica_placement:100 version:3 modified_at_second:1614877294 - volume id:268 size:1074490592 collection:"collection2" file_count:21698 delete_count:1 deleted_byte_count:33968 replica_placement:100 version:3 modified_at_second:1614877434 - volume id:269 size:1077552872 collection:"collection2" file_count:21875 delete_count:4 deleted_byte_count:347272 replica_placement:100 version:3 modified_at_second:1614877481 - volume id:270 size:1076876568 collection:"collection2" file_count:22057 delete_count:1 deleted_byte_count:43916 replica_placement:100 version:3 modified_at_second:1614877469 - volume id:275 size:1078349024 collection:"collection2" file_count:20808 delete_count:1 deleted_byte_count:1118 replica_placement:100 version:3 modified_at_second:1614884147 - volume id:277 size:1074956288 collection:"collection2" file_count:19260 delete_count:2 deleted_byte_count:172356 replica_placement:100 version:3 modified_at_second:1614889988 - volume id:278 size:1078798640 collection:"collection2" file_count:20597 delete_count:5 deleted_byte_count:400060 replica_placement:100 version:3 modified_at_second:1614890292 - volume id:279 size:1077325040 collection:"collection2" file_count:19671 delete_count:6 deleted_byte_count:379116 replica_placement:100 version:3 modified_at_second:1614890229 - volume id:280 size:1077432216 collection:"collection2" file_count:20286 delete_count:1 deleted_byte_count:879 replica_placement:100 version:3 modified_at_second:1614890262 - volume id:281 size:1077581096 collection:"collection2" file_count:20206 delete_count:3 deleted_byte_count:143964 replica_placement:100 version:3 modified_at_second:1614890237 - volume id:284 size:1074533384 collection:"collection2" file_count:22196 delete_count:4 deleted_byte_count:154683 replica_placement:100 version:3 modified_at_second:1614897231 - volume id:285 size:1082128688 collection:"collection2" file_count:21804 delete_count:1 deleted_byte_count:1064 replica_placement:100 version:3 modified_at_second:1614897165 - volume id:289 size:1075284256 collection:"collection2" file_count:29342 delete_count:5 deleted_byte_count:100454 replica_placement:100 version:3 modified_at_second:1614904977 - volume id:290 size:1074723792 collection:"collection2" file_count:28340 delete_count:4 deleted_byte_count:199064 replica_placement:100 version:3 modified_at_second:1614904924 - volume id:291 size:1076256768 collection:"collection2" file_count:28301 delete_count:5 deleted_byte_count:116027 replica_placement:100 version:3 modified_at_second:1614904924 - volume id:293 size:1075409792 collection:"collection2" file_count:26063 delete_count:4 deleted_byte_count:183834 replica_placement:100 version:3 modified_at_second:1614912235 - volume id:294 size:1075444048 collection:"collection2" file_count:26076 delete_count:4 deleted_byte_count:194914 replica_placement:100 version:3 modified_at_second:1614912220 - volume id:296 size:1077824032 collection:"collection2" file_count:26741 delete_count:4 deleted_byte_count:199906 replica_placement:100 version:3 modified_at_second:1614912301 - volume id:297 size:1080229136 collection:"collection2" file_count:23409 delete_count:5 deleted_byte_count:46268 replica_placement:100 version:3 modified_at_second:1614918481 - volume id:298 size:1075410136 collection:"collection2" file_count:23222 delete_count:2 deleted_byte_count:46110 replica_placement:100 version:3 modified_at_second:1614918474 - volume id:299 size:1075147936 collection:"collection2" file_count:22927 delete_count:4 deleted_byte_count:345569 replica_placement:100 version:3 modified_at_second:1614918455 - volume id:300 size:1076212392 collection:"collection2" file_count:22892 delete_count:2 deleted_byte_count:61320 replica_placement:100 version:3 modified_at_second:1614918464 - volume id:301 size:1074655600 collection:"collection2" file_count:22543 delete_count:6 deleted_byte_count:136968 replica_placement:100 version:3 modified_at_second:1614918378 - volume id:303 size:1075944480 collection:"collection2" file_count:22541 delete_count:2 deleted_byte_count:13617 replica_placement:100 version:3 modified_at_second:1614925431 - volume id:306 size:1074764016 collection:"collection2" file_count:22939 replica_placement:100 version:3 modified_at_second:1614925462 - volume id:307 size:1076568000 collection:"collection2" file_count:23377 delete_count:2 deleted_byte_count:25453 replica_placement:100 version:3 modified_at_second:1614931448 - volume id:308 size:1074022392 collection:"collection2" file_count:23086 delete_count:2 deleted_byte_count:2127 replica_placement:100 version:3 modified_at_second:1614931401 - volume id:309 size:1075178664 collection:"collection2" file_count:22692 delete_count:3 deleted_byte_count:171124 replica_placement:100 version:3 modified_at_second:1614931409 - volume id:310 size:1074761528 collection:"collection2" file_count:21441 delete_count:3 deleted_byte_count:13934 replica_placement:100 version:3 modified_at_second:1614931077 - volume id:314 size:1074670840 collection:"collection2" file_count:20964 delete_count:4 deleted_byte_count:304291 replica_placement:100 version:3 modified_at_second:1614937441 - volume id:315 size:1084153544 collection:"collection2" file_count:23638 delete_count:2 deleted_byte_count:53956 replica_placement:100 version:3 modified_at_second:1614937885 - volume id:317 size:1076215096 collection:"collection2" file_count:23572 delete_count:2 deleted_byte_count:1441356 replica_placement:100 version:3 modified_at_second:1614943965 - volume id:318 size:1075965168 collection:"collection2" file_count:22459 delete_count:2 deleted_byte_count:37778 replica_placement:100 version:3 modified_at_second:1614943862 - volume id:319 size:1073952880 collection:"collection2" file_count:22286 delete_count:2 deleted_byte_count:43421 replica_placement:100 version:3 modified_at_second:1614943810 - volume id:320 size:1082437792 collection:"collection2" file_count:21544 delete_count:3 deleted_byte_count:16712 replica_placement:100 version:3 modified_at_second:1614943599 - volume id:321 size:1081477904 collection:"collection2" file_count:23531 delete_count:5 deleted_byte_count:262564 replica_placement:100 version:3 modified_at_second:1614943982 - volume id:324 size:1075606680 collection:"collection2" file_count:20799 delete_count:1 deleted_byte_count:251210 replica_placement:100 version:3 modified_at_second:1614950310 - volume id:325 size:1080701144 collection:"collection2" file_count:21735 replica_placement:100 version:3 modified_at_second:1614950525 - volume id:330 size:1080825832 collection:"collection2" file_count:22464 delete_count:2 deleted_byte_count:15771 replica_placement:100 version:3 modified_at_second:1614956477 - volume id:332 size:1075569928 collection:"collection2" file_count:22097 delete_count:3 deleted_byte_count:98273 replica_placement:100 version:3 modified_at_second:1614962869 - volume id:334 size:1075607880 collection:"collection2" file_count:22546 delete_count:6 deleted_byte_count:101538 replica_placement:100 version:3 modified_at_second:1614962978 - volume id:336 size:1087853056 collection:"collection2" file_count:22801 delete_count:2 deleted_byte_count:26394 replica_placement:100 version:3 modified_at_second:1614963005 - volume id:337 size:1075646784 collection:"collection2" file_count:21934 delete_count:1 deleted_byte_count:3397 replica_placement:100 version:3 modified_at_second:1614969937 - volume id:338 size:1076118304 collection:"collection2" file_count:21680 replica_placement:100 version:3 modified_at_second:1614969850 - volume id:340 size:1079462184 collection:"collection2" file_count:22319 delete_count:4 deleted_byte_count:93620 replica_placement:100 version:3 modified_at_second:1614969978 - volume id:341 size:1074448400 collection:"collection2" file_count:21590 delete_count:5 deleted_byte_count:160085 replica_placement:100 version:3 modified_at_second:1614969858 - volume id:342 size:1080186424 collection:"collection2" file_count:22405 delete_count:4 deleted_byte_count:64819 replica_placement:100 version:3 modified_at_second:1614977521 - volume id:344 size:1075035416 collection:"collection2" file_count:21765 delete_count:1 deleted_byte_count:24623 replica_placement:100 version:3 modified_at_second:1614977465 - volume id:345 size:1074560760 collection:"collection2" file_count:22117 delete_count:2 deleted_byte_count:373286 replica_placement:100 version:3 modified_at_second:1614977457 - volume id:346 size:1076464112 collection:"collection2" file_count:22320 delete_count:4 deleted_byte_count:798258 replica_placement:100 version:3 modified_at_second:1615631322 - volume id:348 size:1080623640 collection:"collection2" file_count:21667 delete_count:1 deleted_byte_count:2443 replica_placement:100 version:3 modified_at_second:1614984606 - volume id:350 size:1074756688 collection:"collection2" file_count:21990 delete_count:3 deleted_byte_count:233881 replica_placement:100 version:3 modified_at_second:1614984682 - volume id:351 size:1078795112 collection:"collection2" file_count:23660 delete_count:3 deleted_byte_count:102141 replica_placement:100 version:3 modified_at_second:1614984816 - volume id:352 size:1077145936 collection:"collection2" file_count:22066 delete_count:1 deleted_byte_count:1018 replica_placement:100 version:3 modified_at_second:1614992130 - volume id:353 size:1074897496 collection:"collection2" file_count:21266 delete_count:2 deleted_byte_count:3105374 replica_placement:100 version:3 modified_at_second:1614991951 - volume id:355 size:1075239728 collection:"collection2" file_count:22244 delete_count:1 deleted_byte_count:23282 replica_placement:100 version:3 modified_at_second:1614992157 - volume id:356 size:1083305048 collection:"collection2" file_count:21552 delete_count:4 deleted_byte_count:14472 replica_placement:100 version:3 modified_at_second:1614992028 - volume id:358 size:1085152368 collection:"collection2" file_count:23756 delete_count:3 deleted_byte_count:44531 replica_placement:100 version:3 modified_at_second:1614998824 - volume id:360 size:1075532456 collection:"collection2" file_count:22574 delete_count:3 deleted_byte_count:1774776 replica_placement:100 version:3 modified_at_second:1614998770 - volume id:361 size:1075362744 collection:"collection2" file_count:22272 delete_count:1 deleted_byte_count:3497 replica_placement:100 version:3 modified_at_second:1614998668 - volume id:375 size:1076140568 collection:"collection2" file_count:21880 delete_count:2 deleted_byte_count:51103 replica_placement:100 version:3 modified_at_second:1615016787 - volume id:376 size:1074845944 collection:"collection2" file_count:22908 delete_count:4 deleted_byte_count:432305 replica_placement:100 version:3 modified_at_second:1615016916 - volume id:377 size:957284144 collection:"collection2" file_count:14923 delete_count:1 deleted_byte_count:1797 replica_placement:100 version:3 modified_at_second:1615632323 - volume id:378 size:959273936 collection:"collection2" file_count:15027 delete_count:4 deleted_byte_count:231414 replica_placement:100 version:3 modified_at_second:1615632323 - volume id:381 size:939261032 collection:"collection2" file_count:14615 delete_count:5 deleted_byte_count:1192272 replica_placement:100 version:3 modified_at_second:1615632324 - Disk hdd total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585 - DataNode 192.168.1.6:8080 total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585 - Rack rack3 total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585 - DataCenter dc3 total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585 - DataCenter dc4 hdd(volume:267/2000 active:267 free:1733 remote:0) - Rack DefaultRack hdd(volume:267/2000 active:267 free:1733 remote:0) - DataNode 192.168.1.1:8080 hdd(volume:267/2000 active:267 free:1733 remote:0) - Disk hdd(volume:267/2000 active:267 free:1733 remote:0) - volume id:1 size:284693256 file_count:1558 delete_count:2 deleted_byte_count:4818 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632062 - volume id:2 size:289228560 file_count:1640 delete_count:4 deleted_byte_count:464508 replica_placement:100 version:3 compact_revision:6 modified_at_second:1615630622 - volume id:3 size:308741952 file_count:1637 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632242 - volume id:4 size:285986968 file_count:1640 delete_count:1 deleted_byte_count:145095 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632302 - volume id:5 size:293806008 file_count:1669 delete_count:2 deleted_byte_count:274334 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631342 - volume id:6 size:302411024 file_count:1604 delete_count:2 deleted_byte_count:274587 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631402 - volume id:7 size:1924728 collection:"collection4" file_count:15 replica_placement:100 version:3 modified_at_second:1609331040 - volume id:9 size:77337416 collection:"collection3" file_count:58 replica_placement:100 version:3 ttl:772 modified_at_second:1615513762 - volume id:10 size:1212784656 collection:"collection0" file_count:58 replica_placement:100 version:3 modified_at_second:1609814543 - volume id:11 size:1109224552 collection:"collection0" file_count:44 replica_placement:100 version:3 modified_at_second:1609815123 - volume id:12 size:1110923848 collection:"collection0" file_count:45 replica_placement:100 version:3 modified_at_second:1609819726 - volume id:13 size:1184910656 collection:"collection0" file_count:47 replica_placement:100 version:3 modified_at_second:1609827832 - volume id:14 size:1107475720 collection:"collection0" file_count:80 delete_count:3 deleted_byte_count:6870 replica_placement:100 version:3 modified_at_second:1612956983 - volume id:15 size:1115965160 collection:"collection0" file_count:83 delete_count:3 deleted_byte_count:4956 replica_placement:100 version:3 modified_at_second:1612957001 - volume id:16 size:1113666048 collection:"collection0" file_count:73 delete_count:5 deleted_byte_count:6318 replica_placement:100 version:3 modified_at_second:1612957007 - volume id:17 size:1095115800 collection:"collection0" file_count:83 delete_count:3 deleted_byte_count:7099 replica_placement:100 version:3 modified_at_second:1612957000 - volume id:18 size:1096678688 collection:"collection0" file_count:88 delete_count:4 deleted_byte_count:8633 replica_placement:100 version:3 modified_at_second:1612957000 - volume id:19 size:1096923792 collection:"collection0" file_count:100 delete_count:10 deleted_byte_count:75779917 replica_placement:100 version:3 compact_revision:4 modified_at_second:1612957011 - volume id:20 size:1074760432 collection:"collection0" file_count:82 delete_count:5 deleted_byte_count:12156 replica_placement:100 version:3 compact_revision:2 modified_at_second:1612957011 - volume id:22 size:1086828368 collection:"collection0" file_count:75 delete_count:3 deleted_byte_count:5551 replica_placement:100 version:3 modified_at_second:1612957007 - volume id:23 size:1076380280 collection:"collection0" file_count:68 delete_count:2 deleted_byte_count:2910 replica_placement:100 version:3 modified_at_second:1612957011 - volume id:24 size:1074139808 collection:"collection0" file_count:90 delete_count:1 deleted_byte_count:1977 replica_placement:100 version:3 modified_at_second:1612957011 - volume id:25 size:690757544 collection:"collection0" file_count:38 delete_count:1 deleted_byte_count:1944 replica_placement:100 version:3 modified_at_second:1612956995 - volume id:26 size:532657632 collection:"collection0" file_count:100 delete_count:4 deleted_byte_count:9081 replica_placement:100 version:3 modified_at_second:1614170023 - volume id:34 size:1077111136 collection:"collection1" file_count:9781 delete_count:110 deleted_byte_count:20894827 replica_placement:100 version:3 modified_at_second:1615619366 - volume id:35 size:1075241656 collection:"collection1" file_count:10523 delete_count:96 deleted_byte_count:46618989 replica_placement:100 version:3 modified_at_second:1615618790 - volume id:36 size:1075118360 collection:"collection1" file_count:10342 delete_count:116 deleted_byte_count:25493106 replica_placement:100 version:3 modified_at_second:1615606148 - volume id:37 size:1075895584 collection:"collection1" file_count:12013 delete_count:98 deleted_byte_count:50747932 replica_placement:100 version:3 modified_at_second:1615594777 - volume id:39 size:1076606536 collection:"collection1" file_count:12612 delete_count:78 deleted_byte_count:17462730 replica_placement:100 version:3 modified_at_second:1615611959 - volume id:40 size:1075358552 collection:"collection1" file_count:12597 delete_count:62 deleted_byte_count:11657901 replica_placement:100 version:3 modified_at_second:1615612994 - volume id:41 size:1076283528 collection:"collection1" file_count:12088 delete_count:84 deleted_byte_count:19319268 replica_placement:100 version:3 modified_at_second:1615596736 - volume id:42 size:1093948352 collection:"collection1" file_count:7889 delete_count:47 deleted_byte_count:5697275 replica_placement:100 version:3 modified_at_second:1615548908 - volume id:43 size:1116445864 collection:"collection1" file_count:7358 delete_count:54 deleted_byte_count:9534379 replica_placement:100 version:3 modified_at_second:1615566170 - volume id:44 size:1077582560 collection:"collection1" file_count:7295 delete_count:50 deleted_byte_count:12618414 replica_placement:100 version:3 modified_at_second:1615566170 - volume id:45 size:1075254640 collection:"collection1" file_count:10772 delete_count:76 deleted_byte_count:22426345 replica_placement:100 version:3 modified_at_second:1615573499 - volume id:46 size:1075286056 collection:"collection1" file_count:9947 delete_count:309 deleted_byte_count:105601163 replica_placement:100 version:3 modified_at_second:1615569826 - volume id:48 size:1076778720 collection:"collection1" file_count:9850 delete_count:77 deleted_byte_count:16641907 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630690 - volume id:50 size:1076688224 collection:"collection1" file_count:7921 delete_count:26 deleted_byte_count:5162032 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615610879 - volume id:52 size:1083529704 collection:"collection1" file_count:10128 delete_count:32 deleted_byte_count:10608391 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615599195 - volume id:53 size:1063089216 collection:"collection1" file_count:9832 delete_count:31 deleted_byte_count:9273066 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632156 - volume id:55 size:1012890016 collection:"collection1" file_count:8651 delete_count:27 deleted_byte_count:9418841 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631452 - volume id:57 size:839849792 collection:"collection1" file_count:7514 delete_count:24 deleted_byte_count:6228543 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631774 - volume id:58 size:908064200 collection:"collection1" file_count:8128 delete_count:21 deleted_byte_count:6113731 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632342 - volume id:59 size:988302272 collection:"collection1" file_count:8098 delete_count:20 deleted_byte_count:3947615 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632238 - volume id:60 size:1010702480 collection:"collection1" file_count:8969 delete_count:79 deleted_byte_count:24782814 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632439 - volume id:61 size:975604488 collection:"collection1" file_count:8683 delete_count:20 deleted_byte_count:10276072 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615631176 - volume id:62 size:873845936 collection:"collection1" file_count:7897 delete_count:23 deleted_byte_count:10920170 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631133 - volume id:64 size:965638488 collection:"collection1" file_count:8218 delete_count:27 deleted_byte_count:6922489 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631031 - volume id:65 size:823283552 collection:"collection1" file_count:7834 delete_count:29 deleted_byte_count:5950610 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632306 - volume id:66 size:821343440 collection:"collection1" file_count:7383 delete_count:29 deleted_byte_count:12010343 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631968 - volume id:67 size:878713872 collection:"collection1" file_count:7299 delete_count:117 deleted_byte_count:24857326 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632156 - volume id:68 size:898630584 collection:"collection1" file_count:6934 delete_count:95 deleted_byte_count:27460707 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632284 - volume id:70 size:886695472 collection:"collection1" file_count:7769 delete_count:164 deleted_byte_count:45162513 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632398 - volume id:71 size:907608392 collection:"collection1" file_count:7658 delete_count:122 deleted_byte_count:27622941 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632307 - volume id:72 size:903990720 collection:"collection1" file_count:6996 delete_count:240 deleted_byte_count:74147727 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630982 - volume id:73 size:929047664 collection:"collection1" file_count:7038 delete_count:227 deleted_byte_count:65336664 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630707 - volume id:74 size:957046128 collection:"collection1" file_count:6981 delete_count:259 deleted_byte_count:73080838 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631460 - volume id:75 size:908044992 collection:"collection1" file_count:6911 delete_count:268 deleted_byte_count:73934373 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632430 - volume id:76 size:985296744 collection:"collection1" file_count:6566 delete_count:61 deleted_byte_count:44464430 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632284 - volume id:77 size:929398296 collection:"collection1" file_count:7427 delete_count:238 deleted_byte_count:59581579 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632013 - volume id:78 size:1075671512 collection:"collection1" file_count:7540 delete_count:258 deleted_byte_count:71726846 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615582829 - volume id:79 size:948225472 collection:"collection1" file_count:6997 delete_count:227 deleted_byte_count:60625763 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631326 - volume id:82 size:1041661800 collection:"collection1" file_count:7043 delete_count:207 deleted_byte_count:52275724 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632430 - volume id:83 size:936195856 collection:"collection1" file_count:7593 delete_count:13 deleted_byte_count:4633917 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632029 - volume id:85 size:1023867520 collection:"collection1" file_count:7787 delete_count:240 deleted_byte_count:82091742 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631723 - volume id:86 size:1009437488 collection:"collection1" file_count:8474 delete_count:236 deleted_byte_count:64543674 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630812 - volume id:87 size:922276640 collection:"collection1" file_count:12902 delete_count:13 deleted_byte_count:3412959 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632438 - volume id:89 size:1044401976 collection:"collection1" file_count:14943 delete_count:243 deleted_byte_count:58543159 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632208 - volume id:90 size:891145784 collection:"collection1" file_count:14608 delete_count:10 deleted_byte_count:2564369 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615629390 - volume id:91 size:936572832 collection:"collection1" file_count:14686 delete_count:11 deleted_byte_count:4717727 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631851 - volume id:92 size:992440712 collection:"collection1" file_count:7061 delete_count:195 deleted_byte_count:60649573 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630566 - volume id:93 size:1079603768 collection:"collection1" file_count:7878 delete_count:270 deleted_byte_count:74150048 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615556015 - volume id:94 size:1030685824 collection:"collection1" file_count:7660 delete_count:207 deleted_byte_count:70150733 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631616 - volume id:95 size:990879168 collection:"collection1" file_count:6620 delete_count:206 deleted_byte_count:60363604 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615631866 - volume id:96 size:989296136 collection:"collection1" file_count:7544 delete_count:229 deleted_byte_count:59931853 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630778 - volume id:97 size:1053112992 collection:"collection1" file_count:6789 delete_count:50 deleted_byte_count:38894001 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631194 - volume id:99 size:1071718504 collection:"collection1" file_count:7470 delete_count:8 deleted_byte_count:9624950 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631175 - volume id:100 size:1083617440 collection:"collection1" file_count:7018 delete_count:187 deleted_byte_count:61304236 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615505917 - volume id:101 size:1077109520 collection:"collection1" file_count:7706 delete_count:226 deleted_byte_count:77864841 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630994 - volume id:102 size:1074359920 collection:"collection1" file_count:7338 delete_count:7 deleted_byte_count:6499151 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615626683 - volume id:103 size:1075863904 collection:"collection1" file_count:7184 delete_count:186 deleted_byte_count:58872238 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615628417 - volume id:104 size:1076383768 collection:"collection1" file_count:7663 delete_count:184 deleted_byte_count:100578087 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615602661 - volume id:105 size:1073996824 collection:"collection1" file_count:6873 delete_count:19 deleted_byte_count:14271533 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615499756 - volume id:108 size:1074648024 collection:"collection1" file_count:7472 delete_count:194 deleted_byte_count:70864699 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615593232 - volume id:109 size:1075254560 collection:"collection1" file_count:7556 delete_count:263 deleted_byte_count:55155265 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615502487 - volume id:110 size:1076575744 collection:"collection1" file_count:6996 delete_count:163 deleted_byte_count:52954032 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615590786 - volume id:111 size:1073826232 collection:"collection1" file_count:7355 delete_count:155 deleted_byte_count:50083578 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615593233 - volume id:114 size:1074762784 collection:"collection1" file_count:8802 delete_count:156 deleted_byte_count:38470055 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615591826 - volume id:115 size:1076192240 collection:"collection1" file_count:7690 delete_count:154 deleted_byte_count:32267193 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615285295 - volume id:116 size:1074489504 collection:"collection1" file_count:9981 delete_count:174 deleted_byte_count:53998777 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615611567 - volume id:117 size:1073917192 collection:"collection1" file_count:9520 delete_count:114 deleted_byte_count:21835126 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615573714 - volume id:118 size:1074064400 collection:"collection1" file_count:8738 delete_count:15 deleted_byte_count:3460697 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615516265 - volume id:119 size:1075940104 collection:"collection1" file_count:9003 delete_count:12 deleted_byte_count:9128155 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615573880 - volume id:120 size:1076115928 collection:"collection1" file_count:9639 delete_count:118 deleted_byte_count:33357871 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615482567 - volume id:121 size:1078803248 collection:"collection1" file_count:10113 delete_count:441 deleted_byte_count:94128627 replica_placement:100 version:3 modified_at_second:1615506629 - volume id:122 size:1076235312 collection:"collection1" file_count:9106 delete_count:252 deleted_byte_count:93041272 replica_placement:100 version:3 modified_at_second:1615585913 - volume id:123 size:1080491112 collection:"collection1" file_count:10623 delete_count:302 deleted_byte_count:83956998 replica_placement:100 version:3 modified_at_second:1615585916 - volume id:124 size:1074519360 collection:"collection1" file_count:9457 delete_count:286 deleted_byte_count:74752459 replica_placement:100 version:3 modified_at_second:1615585916 - volume id:125 size:1088687040 collection:"collection1" file_count:9518 delete_count:281 deleted_byte_count:76037905 replica_placement:100 version:3 modified_at_second:1615585913 - volume id:126 size:1073867464 collection:"collection1" file_count:9320 delete_count:278 deleted_byte_count:94547424 replica_placement:100 version:3 modified_at_second:1615585912 - volume id:127 size:1074907336 collection:"collection1" file_count:9900 delete_count:133 deleted_byte_count:48570820 replica_placement:100 version:3 modified_at_second:1615612991 - volume id:129 size:1074704272 collection:"collection1" file_count:10012 delete_count:150 deleted_byte_count:64491721 replica_placement:100 version:3 modified_at_second:1615627566 - volume id:130 size:1075000632 collection:"collection1" file_count:10633 delete_count:161 deleted_byte_count:34768201 replica_placement:100 version:3 modified_at_second:1615582330 - volume id:131 size:1075279584 collection:"collection1" file_count:10075 delete_count:135 deleted_byte_count:29795712 replica_placement:100 version:3 modified_at_second:1615523898 - volume id:132 size:1088539496 collection:"collection1" file_count:11051 delete_count:71 deleted_byte_count:17178322 replica_placement:100 version:3 modified_at_second:1615619584 - volume id:133 size:1075952760 collection:"collection1" file_count:9538 delete_count:74 deleted_byte_count:19558008 replica_placement:100 version:3 modified_at_second:1615584780 - volume id:134 size:1074367304 collection:"collection1" file_count:10662 delete_count:69 deleted_byte_count:25530139 replica_placement:100 version:3 modified_at_second:1615555876 - volume id:135 size:1073906720 collection:"collection1" file_count:10446 delete_count:71 deleted_byte_count:28599975 replica_placement:100 version:3 modified_at_second:1615569816 - volume id:137 size:1074309264 collection:"collection1" file_count:9633 delete_count:50 deleted_byte_count:27487972 replica_placement:100 version:3 modified_at_second:1615572231 - volume id:139 size:1074163936 collection:"collection1" file_count:9314 delete_count:43 deleted_byte_count:10631353 replica_placement:100 version:3 modified_at_second:1615571946 - volume id:141 size:1074619488 collection:"collection1" file_count:9840 delete_count:45 deleted_byte_count:40890181 replica_placement:100 version:3 modified_at_second:1615630994 - volume id:142 size:1075732992 collection:"collection1" file_count:9009 delete_count:48 deleted_byte_count:9912854 replica_placement:100 version:3 modified_at_second:1615598914 - volume id:143 size:1075011280 collection:"collection1" file_count:9608 delete_count:51 deleted_byte_count:37282460 replica_placement:100 version:3 modified_at_second:1615488586 - volume id:145 size:1074394928 collection:"collection1" file_count:9255 delete_count:34 deleted_byte_count:38011392 replica_placement:100 version:3 modified_at_second:1615591825 - volume id:146 size:1076337520 collection:"collection1" file_count:10492 delete_count:50 deleted_byte_count:17071505 replica_placement:100 version:3 modified_at_second:1615632005 - volume id:147 size:1077130544 collection:"collection1" file_count:10451 delete_count:27 deleted_byte_count:8290907 replica_placement:100 version:3 modified_at_second:1615604117 - volume id:148 size:1076066568 collection:"collection1" file_count:9547 delete_count:33 deleted_byte_count:7034089 replica_placement:100 version:3 modified_at_second:1615586393 - volume id:149 size:1074989016 collection:"collection1" file_count:8352 delete_count:35 deleted_byte_count:7179742 replica_placement:100 version:3 modified_at_second:1615494496 - volume id:150 size:1076290408 collection:"collection1" file_count:9328 delete_count:33 deleted_byte_count:43417791 replica_placement:100 version:3 modified_at_second:1615611569 - volume id:151 size:1098659752 collection:"collection1" file_count:10805 delete_count:27 deleted_byte_count:7209106 replica_placement:100 version:3 modified_at_second:1615586390 - volume id:152 size:1075941376 collection:"collection1" file_count:9951 delete_count:36 deleted_byte_count:25348335 replica_placement:100 version:3 modified_at_second:1615606614 - volume id:153 size:1078539784 collection:"collection1" file_count:10924 delete_count:34 deleted_byte_count:12603081 replica_placement:100 version:3 modified_at_second:1615606614 - volume id:154 size:1081244752 collection:"collection1" file_count:11002 delete_count:31 deleted_byte_count:8467560 replica_placement:100 version:3 modified_at_second:1615478471 - volume id:156 size:1074975832 collection:"collection1" file_count:9535 delete_count:40 deleted_byte_count:11426621 replica_placement:100 version:3 modified_at_second:1615628342 - volume id:157 size:1076758536 collection:"collection1" file_count:10012 delete_count:19 deleted_byte_count:11688737 replica_placement:100 version:3 modified_at_second:1615597782 - volume id:158 size:1087251976 collection:"collection1" file_count:9972 delete_count:20 deleted_byte_count:10328429 replica_placement:100 version:3 modified_at_second:1615588879 - volume id:159 size:1074132336 collection:"collection1" file_count:9382 delete_count:27 deleted_byte_count:11474574 replica_placement:100 version:3 modified_at_second:1615593593 - volume id:160 size:1075680976 collection:"collection1" file_count:9772 delete_count:22 deleted_byte_count:4981968 replica_placement:100 version:3 modified_at_second:1615597782 - volume id:161 size:1077397136 collection:"collection1" file_count:9988 delete_count:28 deleted_byte_count:12509164 replica_placement:100 version:3 modified_at_second:1615631452 - volume id:162 size:1074286880 collection:"collection1" file_count:11220 delete_count:17 deleted_byte_count:1815547 replica_placement:100 version:3 modified_at_second:1615478127 - volume id:163 size:1074457224 collection:"collection1" file_count:12524 delete_count:27 deleted_byte_count:6359619 replica_placement:100 version:3 modified_at_second:1615579313 - volume id:164 size:1074261256 collection:"collection1" file_count:11922 delete_count:25 deleted_byte_count:2923288 replica_placement:100 version:3 modified_at_second:1615620085 - volume id:165 size:1073891080 collection:"collection1" file_count:9152 delete_count:12 deleted_byte_count:19164659 replica_placement:100 version:3 modified_at_second:1615471910 - volume id:166 size:1075637536 collection:"collection1" file_count:14211 delete_count:24 deleted_byte_count:20415490 replica_placement:100 version:3 modified_at_second:1615491021 - volume id:167 size:1073958280 collection:"collection1" file_count:25231 delete_count:48 deleted_byte_count:26022344 replica_placement:100 version:3 modified_at_second:1615620014 - volume id:168 size:1074718864 collection:"collection1" file_count:25702 delete_count:40 deleted_byte_count:4024775 replica_placement:100 version:3 modified_at_second:1615585664 - volume id:169 size:1073863032 collection:"collection1" file_count:25248 delete_count:43 deleted_byte_count:3013817 replica_placement:100 version:3 modified_at_second:1615569832 - volume id:170 size:1075747088 collection:"collection1" file_count:24596 delete_count:41 deleted_byte_count:3494711 replica_placement:100 version:3 modified_at_second:1615579207 - volume id:171 size:1081881400 collection:"collection1" file_count:24215 delete_count:36 deleted_byte_count:3191335 replica_placement:100 version:3 modified_at_second:1615596486 - volume id:172 size:1074787304 collection:"collection1" file_count:31236 delete_count:50 deleted_byte_count:3316482 replica_placement:100 version:3 modified_at_second:1615612385 - volume id:174 size:1073824160 collection:"collection1" file_count:30689 delete_count:36 deleted_byte_count:2160116 replica_placement:100 version:3 modified_at_second:1615598914 - volume id:175 size:1077742472 collection:"collection1" file_count:32353 delete_count:33 deleted_byte_count:1861403 replica_placement:100 version:3 modified_at_second:1615559516 - volume id:176 size:1073854800 collection:"collection1" file_count:30582 delete_count:34 deleted_byte_count:7701976 replica_placement:100 version:3 modified_at_second:1615626169 - volume id:178 size:1087560112 collection:"collection1" file_count:23482 delete_count:22 deleted_byte_count:18810492 replica_placement:100 version:3 modified_at_second:1615541369 - volume id:179 size:1074313920 collection:"collection1" file_count:21829 delete_count:24 deleted_byte_count:45574435 replica_placement:100 version:3 modified_at_second:1615580308 - volume id:180 size:1078438448 collection:"collection1" file_count:23614 delete_count:12 deleted_byte_count:4496474 replica_placement:100 version:3 modified_at_second:1614773243 - volume id:181 size:1074571672 collection:"collection1" file_count:22898 delete_count:19 deleted_byte_count:6628413 replica_placement:100 version:3 modified_at_second:1614745117 - volume id:183 size:1076361616 collection:"collection1" file_count:31293 delete_count:16 deleted_byte_count:468841 replica_placement:100 version:3 modified_at_second:1615572985 - volume id:184 size:1074594216 collection:"collection1" file_count:31368 delete_count:22 deleted_byte_count:857453 replica_placement:100 version:3 modified_at_second:1615586578 - volume id:185 size:1074099592 collection:"collection1" file_count:30612 delete_count:17 deleted_byte_count:2610847 replica_placement:100 version:3 modified_at_second:1615506835 - volume id:186 size:1074220664 collection:"collection1" file_count:31450 delete_count:15 deleted_byte_count:391855 replica_placement:100 version:3 modified_at_second:1615614934 - volume id:187 size:1074396112 collection:"collection1" file_count:31853 delete_count:17 deleted_byte_count:454283 replica_placement:100 version:3 modified_at_second:1615590491 - volume id:188 size:1074732632 collection:"collection1" file_count:31867 delete_count:19 deleted_byte_count:393743 replica_placement:100 version:3 modified_at_second:1615487645 - volume id:189 size:1074847824 collection:"collection1" file_count:31450 delete_count:16 deleted_byte_count:1040552 replica_placement:100 version:3 modified_at_second:1615335661 - volume id:190 size:1074008968 collection:"collection1" file_count:31987 delete_count:11 deleted_byte_count:685125 replica_placement:100 version:3 modified_at_second:1615447162 - volume id:191 size:1075492960 collection:"collection1" file_count:31301 delete_count:19 deleted_byte_count:708401 replica_placement:100 version:3 modified_at_second:1615357457 - volume id:192 size:1075857384 collection:"collection1" file_count:31490 delete_count:25 deleted_byte_count:720617 replica_placement:100 version:3 modified_at_second:1615621632 - volume id:193 size:1076616760 collection:"collection1" file_count:31907 delete_count:16 deleted_byte_count:464900 replica_placement:100 version:3 modified_at_second:1615507877 - volume id:194 size:1073985792 collection:"collection1" file_count:31434 delete_count:18 deleted_byte_count:391432 replica_placement:100 version:3 modified_at_second:1615559502 - volume id:195 size:1074158304 collection:"collection1" file_count:31453 delete_count:15 deleted_byte_count:718266 replica_placement:100 version:3 modified_at_second:1615559331 - volume id:196 size:1074594640 collection:"collection1" file_count:31665 delete_count:18 deleted_byte_count:3468922 replica_placement:100 version:3 modified_at_second:1615501690 - volume id:198 size:1075104624 collection:"collection1" file_count:16577 delete_count:18 deleted_byte_count:6583181 replica_placement:100 version:3 modified_at_second:1615623371 - volume id:199 size:1078117688 collection:"collection1" file_count:16497 delete_count:14 deleted_byte_count:1514286 replica_placement:100 version:3 modified_at_second:1615585987 - volume id:200 size:1075630464 collection:"collection1" file_count:16380 delete_count:18 deleted_byte_count:1103109 replica_placement:100 version:3 modified_at_second:1615485252 - volume id:201 size:1091460440 collection:"collection1" file_count:16684 delete_count:26 deleted_byte_count:5590335 replica_placement:100 version:3 modified_at_second:1615585987 - volume id:204 size:1079766904 collection:"collection1" file_count:3233 delete_count:255 deleted_byte_count:104707641 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615565702 - volume id:207 size:1081939960 collection:"collection1" file_count:3010 delete_count:4 deleted_byte_count:692350 replica_placement:100 version:3 modified_at_second:1615269061 - volume id:208 size:1077863624 collection:"collection1" file_count:3147 delete_count:6 deleted_byte_count:858726 replica_placement:100 version:3 modified_at_second:1615495515 - volume id:209 size:1074083592 collection:"collection1" file_count:3238 delete_count:4 deleted_byte_count:1494244 replica_placement:100 version:3 modified_at_second:1615419954 - volume id:210 size:1094311304 collection:"collection1" file_count:3468 delete_count:4 deleted_byte_count:466433 replica_placement:100 version:3 modified_at_second:1615495515 - volume id:211 size:1080610712 collection:"collection1" file_count:3247 delete_count:7 deleted_byte_count:1891456 replica_placement:100 version:3 modified_at_second:1615269124 - volume id:216 size:1080073496 collection:"collection1" file_count:3316 delete_count:4 deleted_byte_count:179819 replica_placement:100 version:3 modified_at_second:1615586387 - volume id:218 size:1081263944 collection:"collection1" file_count:3433 delete_count:14 deleted_byte_count:3454237 replica_placement:100 version:3 modified_at_second:1615603637 - volume id:220 size:1081928312 collection:"collection1" file_count:3166 delete_count:13 deleted_byte_count:4127709 replica_placement:100 version:3 modified_at_second:1615579317 - volume id:221 size:1106545536 collection:"collection1" file_count:3153 delete_count:11 deleted_byte_count:1496835 replica_placement:100 version:3 modified_at_second:1615269138 - volume id:224 size:1093691520 collection:"collection1" file_count:3463 delete_count:10 deleted_byte_count:1128328 replica_placement:100 version:3 modified_at_second:1615601870 - volume id:225 size:1080698928 collection:"collection1" file_count:3115 delete_count:7 deleted_byte_count:18170416 replica_placement:100 version:3 modified_at_second:1615434685 - volume id:226 size:1103504792 collection:"collection1" file_count:2965 delete_count:10 deleted_byte_count:2639254 replica_placement:100 version:3 modified_at_second:1615601870 - volume id:227 size:1106699864 collection:"collection1" file_count:2827 delete_count:19 deleted_byte_count:5393310 replica_placement:100 version:3 modified_at_second:1615609989 - volume id:228 size:1109784072 collection:"collection1" file_count:2504 delete_count:24 deleted_byte_count:5458950 replica_placement:100 version:3 modified_at_second:1615610489 - volume id:229 size:1109855256 collection:"collection1" file_count:2857 delete_count:22 deleted_byte_count:2839883 replica_placement:100 version:3 modified_at_second:1615609989 - volume id:231 size:1112917664 collection:"collection1" file_count:3151 delete_count:19 deleted_byte_count:2852517 replica_placement:100 version:3 modified_at_second:1615611350 - volume id:232 size:1073901520 collection:"collection1" file_count:3004 delete_count:54 deleted_byte_count:10273081 replica_placement:100 version:3 modified_at_second:1615611352 - volume id:233 size:1080526464 collection:"collection1" file_count:3136 delete_count:61 deleted_byte_count:17991717 replica_placement:100 version:3 modified_at_second:1615611354 - volume id:236 size:1089476200 collection:"collection1" file_count:3231 delete_count:53 deleted_byte_count:11625921 replica_placement:100 version:3 modified_at_second:1615611351 - volume id:238 size:354320000 collection:"collection1" file_count:701 delete_count:17 deleted_byte_count:5940420 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632030 - volume id:240 size:424791528 collection:"collection1" file_count:734 delete_count:12 deleted_byte_count:7353071 replica_placement:100 version:3 modified_at_second:1615631669 - volume id:242 size:1075383304 collection:"collection2" file_count:10470 replica_placement:100 version:3 modified_at_second:1614852115 - volume id:243 size:1088174560 collection:"collection2" file_count:11109 delete_count:1 deleted_byte_count:938 replica_placement:100 version:3 modified_at_second:1614852202 - volume id:245 size:1074597056 collection:"collection2" file_count:10371 delete_count:3 deleted_byte_count:209701 replica_placement:100 version:3 modified_at_second:1614852093 - volume id:247 size:1075859784 collection:"collection2" file_count:10443 delete_count:2 deleted_byte_count:564486 replica_placement:100 version:3 modified_at_second:1614856152 - volume id:249 size:1074819168 collection:"collection2" file_count:10763 delete_count:2 deleted_byte_count:271699 replica_placement:100 version:3 modified_at_second:1614856231 - volume id:250 size:1080572256 collection:"collection2" file_count:10220 replica_placement:100 version:3 modified_at_second:1614856129 - volume id:251 size:1075684408 collection:"collection2" file_count:10847 replica_placement:100 version:3 modified_at_second:1614856270 - volume id:254 size:1074830800 collection:"collection2" file_count:14140 delete_count:2 deleted_byte_count:105892 replica_placement:100 version:3 modified_at_second:1614861115 - volume id:257 size:1082621664 collection:"collection2" file_count:18172 delete_count:2 deleted_byte_count:25125 replica_placement:100 version:3 modified_at_second:1614866395 - volume id:260 size:1075105664 collection:"collection2" file_count:17316 delete_count:4 deleted_byte_count:2015310 replica_placement:100 version:3 modified_at_second:1614866226 - volume id:261 size:1076628592 collection:"collection2" file_count:18355 delete_count:1 deleted_byte_count:1155 replica_placement:100 version:3 modified_at_second:1614866420 - volume id:262 size:1078492464 collection:"collection2" file_count:20390 delete_count:3 deleted_byte_count:287601 replica_placement:100 version:3 modified_at_second:1614871601 - volume id:263 size:1077167440 collection:"collection2" file_count:20227 delete_count:4 deleted_byte_count:97887 replica_placement:100 version:3 modified_at_second:1614871567 - volume id:268 size:1074490592 collection:"collection2" file_count:21698 delete_count:1 deleted_byte_count:33968 replica_placement:100 version:3 modified_at_second:1614877435 - volume id:269 size:1077552720 collection:"collection2" file_count:21875 delete_count:4 deleted_byte_count:347272 replica_placement:100 version:3 modified_at_second:1614877481 - volume id:271 size:1076992648 collection:"collection2" file_count:22640 delete_count:1 deleted_byte_count:30645 replica_placement:100 version:3 modified_at_second:1614877504 - volume id:273 size:1074873432 collection:"collection2" file_count:20511 delete_count:3 deleted_byte_count:46076 replica_placement:100 version:3 modified_at_second:1614884046 - volume id:274 size:1075994128 collection:"collection2" file_count:20997 replica_placement:100 version:3 modified_at_second:1614884113 - volume id:276 size:1076899888 collection:"collection2" file_count:20190 delete_count:1 deleted_byte_count:8798 replica_placement:100 version:3 modified_at_second:1614884003 - volume id:277 size:1074956160 collection:"collection2" file_count:19260 delete_count:2 deleted_byte_count:172356 replica_placement:100 version:3 modified_at_second:1614889988 - volume id:279 size:1077325096 collection:"collection2" file_count:19671 delete_count:6 deleted_byte_count:379116 replica_placement:100 version:3 modified_at_second:1614890230 - volume id:282 size:1075232240 collection:"collection2" file_count:22659 delete_count:4 deleted_byte_count:67915 replica_placement:100 version:3 modified_at_second:1614897304 - volume id:284 size:1074533384 collection:"collection2" file_count:22196 delete_count:4 deleted_byte_count:154683 replica_placement:100 version:3 modified_at_second:1614897231 - volume id:285 size:1082128576 collection:"collection2" file_count:21804 delete_count:1 deleted_byte_count:1064 replica_placement:100 version:3 modified_at_second:1614897165 - volume id:286 size:1077464824 collection:"collection2" file_count:23905 delete_count:6 deleted_byte_count:630577 replica_placement:100 version:3 modified_at_second:1614897401 - volume id:287 size:1074590528 collection:"collection2" file_count:28163 delete_count:5 deleted_byte_count:35727 replica_placement:100 version:3 modified_at_second:1614904874 - volume id:288 size:1075406800 collection:"collection2" file_count:27243 delete_count:2 deleted_byte_count:51519 replica_placement:100 version:3 modified_at_second:1614904738 - volume id:292 size:1092010744 collection:"collection2" file_count:26781 delete_count:5 deleted_byte_count:508910 replica_placement:100 version:3 modified_at_second:1614912327 - volume id:293 size:1075409776 collection:"collection2" file_count:26063 delete_count:4 deleted_byte_count:183834 replica_placement:100 version:3 modified_at_second:1614912235 - volume id:294 size:1075443992 collection:"collection2" file_count:26076 delete_count:4 deleted_byte_count:194914 replica_placement:100 version:3 modified_at_second:1614912220 - volume id:295 size:1074702376 collection:"collection2" file_count:24488 delete_count:3 deleted_byte_count:48555 replica_placement:100 version:3 modified_at_second:1614911929 - volume id:300 size:1076212424 collection:"collection2" file_count:22892 delete_count:2 deleted_byte_count:61320 replica_placement:100 version:3 modified_at_second:1614918464 - volume id:304 size:1081038888 collection:"collection2" file_count:24505 delete_count:2 deleted_byte_count:124447 replica_placement:100 version:3 modified_at_second:1614925567 - volume id:305 size:1074185552 collection:"collection2" file_count:22074 delete_count:5 deleted_byte_count:20221 replica_placement:100 version:3 modified_at_second:1614925312 - volume id:310 size:1074761520 collection:"collection2" file_count:21441 delete_count:3 deleted_byte_count:13934 replica_placement:100 version:3 modified_at_second:1614931077 - volume id:311 size:1088248208 collection:"collection2" file_count:23553 delete_count:6 deleted_byte_count:191716 replica_placement:100 version:3 modified_at_second:1614931460 - volume id:312 size:1075037808 collection:"collection2" file_count:22524 replica_placement:100 version:3 modified_at_second:1614937832 - volume id:313 size:1074876016 collection:"collection2" file_count:22404 delete_count:4 deleted_byte_count:51728 replica_placement:100 version:3 modified_at_second:1614937755 - volume id:314 size:1074670840 collection:"collection2" file_count:20964 delete_count:4 deleted_byte_count:304291 replica_placement:100 version:3 modified_at_second:1614937441 - volume id:315 size:1084153456 collection:"collection2" file_count:23638 delete_count:2 deleted_byte_count:53956 replica_placement:100 version:3 modified_at_second:1614937884 - volume id:316 size:1077720784 collection:"collection2" file_count:22605 delete_count:1 deleted_byte_count:8503 replica_placement:100 version:3 modified_at_second:1614937838 - volume id:317 size:1076215040 collection:"collection2" file_count:23572 delete_count:2 deleted_byte_count:1441356 replica_placement:100 version:3 modified_at_second:1614943965 - volume id:319 size:1073952744 collection:"collection2" file_count:22286 delete_count:2 deleted_byte_count:43421 replica_placement:100 version:3 modified_at_second:1614943810 - volume id:320 size:1082437736 collection:"collection2" file_count:21544 delete_count:3 deleted_byte_count:16712 replica_placement:100 version:3 modified_at_second:1614943591 - volume id:321 size:1081477960 collection:"collection2" file_count:23531 delete_count:5 deleted_byte_count:262564 replica_placement:100 version:3 modified_at_second:1614943982 - volume id:322 size:1078471600 collection:"collection2" file_count:21905 delete_count:3 deleted_byte_count:145002 replica_placement:100 version:3 modified_at_second:1614950574 - volume id:324 size:1075606712 collection:"collection2" file_count:20799 delete_count:1 deleted_byte_count:251210 replica_placement:100 version:3 modified_at_second:1614950310 - volume id:326 size:1076059936 collection:"collection2" file_count:22564 delete_count:2 deleted_byte_count:192886 replica_placement:100 version:3 modified_at_second:1614950619 - volume id:327 size:1076121224 collection:"collection2" file_count:22007 delete_count:3 deleted_byte_count:60358 replica_placement:100 version:3 modified_at_second:1614956487 - volume id:328 size:1074767928 collection:"collection2" file_count:21720 delete_count:3 deleted_byte_count:56429 replica_placement:100 version:3 modified_at_second:1614956362 - volume id:329 size:1076691776 collection:"collection2" file_count:22411 delete_count:5 deleted_byte_count:214092 replica_placement:100 version:3 modified_at_second:1614956485 - volume id:331 size:1074957192 collection:"collection2" file_count:21230 delete_count:4 deleted_byte_count:62145 replica_placement:100 version:3 modified_at_second:1614956259 - volume id:333 size:1074270192 collection:"collection2" file_count:21271 delete_count:2 deleted_byte_count:168122 replica_placement:100 version:3 modified_at_second:1614962697 - volume id:335 size:1076235176 collection:"collection2" file_count:22391 delete_count:3 deleted_byte_count:8838 replica_placement:100 version:3 modified_at_second:1614962970 - volume id:336 size:1087853032 collection:"collection2" file_count:22801 delete_count:2 deleted_byte_count:26394 replica_placement:100 version:3 modified_at_second:1614963003 - volume id:338 size:1076118360 collection:"collection2" file_count:21680 replica_placement:100 version:3 modified_at_second:1614969850 - volume id:342 size:1080186296 collection:"collection2" file_count:22405 delete_count:4 deleted_byte_count:64819 replica_placement:100 version:3 modified_at_second:1614977518 - volume id:343 size:1075345184 collection:"collection2" file_count:21095 delete_count:2 deleted_byte_count:20581 replica_placement:100 version:3 modified_at_second:1614977148 - volume id:349 size:1075957824 collection:"collection2" file_count:22395 delete_count:2 deleted_byte_count:61565 replica_placement:100 version:3 modified_at_second:1614984748 - volume id:350 size:1074756688 collection:"collection2" file_count:21990 delete_count:3 deleted_byte_count:233881 replica_placement:100 version:3 modified_at_second:1614984682 - volume id:354 size:1085213992 collection:"collection2" file_count:23150 delete_count:4 deleted_byte_count:82391 replica_placement:100 version:3 modified_at_second:1614992207 - volume id:356 size:1083304992 collection:"collection2" file_count:21552 delete_count:4 deleted_byte_count:14472 replica_placement:100 version:3 modified_at_second:1614992027 - volume id:358 size:1085152312 collection:"collection2" file_count:23756 delete_count:3 deleted_byte_count:44531 replica_placement:100 version:3 modified_at_second:1614998824 - volume id:359 size:1074211240 collection:"collection2" file_count:22437 delete_count:2 deleted_byte_count:187953 replica_placement:100 version:3 modified_at_second:1614998711 - volume id:362 size:1074074120 collection:"collection2" file_count:20595 delete_count:1 deleted_byte_count:112145 replica_placement:100 version:3 modified_at_second:1615004407 - volume id:363 size:1078859496 collection:"collection2" file_count:23177 delete_count:4 deleted_byte_count:9601 replica_placement:100 version:3 modified_at_second:1615004822 - volume id:364 size:1081280816 collection:"collection2" file_count:22686 delete_count:1 deleted_byte_count:84375 replica_placement:100 version:3 modified_at_second:1615004813 - volume id:365 size:1075736632 collection:"collection2" file_count:22193 delete_count:5 deleted_byte_count:259033 replica_placement:100 version:3 modified_at_second:1615004776 - volume id:366 size:1075267272 collection:"collection2" file_count:21856 delete_count:5 deleted_byte_count:138363 replica_placement:100 version:3 modified_at_second:1615004703 - volume id:367 size:1076403648 collection:"collection2" file_count:22995 delete_count:2 deleted_byte_count:36955 replica_placement:100 version:3 modified_at_second:1615010985 - volume id:368 size:1074822016 collection:"collection2" file_count:22252 delete_count:4 deleted_byte_count:3291946 replica_placement:100 version:3 modified_at_second:1615010878 - volume id:369 size:1091472040 collection:"collection2" file_count:23709 delete_count:4 deleted_byte_count:400876 replica_placement:100 version:3 modified_at_second:1615011019 - volume id:370 size:1076040480 collection:"collection2" file_count:22092 delete_count:2 deleted_byte_count:115388 replica_placement:100 version:3 modified_at_second:1615010877 - volume id:371 size:1078806160 collection:"collection2" file_count:22685 delete_count:2 deleted_byte_count:68905 replica_placement:100 version:3 modified_at_second:1615010994 - volume id:372 size:1076193312 collection:"collection2" file_count:22774 delete_count:1 deleted_byte_count:3495 replica_placement:100 version:3 modified_at_second:1615016911 - volume id:374 size:1085011080 collection:"collection2" file_count:23054 delete_count:2 deleted_byte_count:89034 replica_placement:100 version:3 modified_at_second:1615016917 - volume id:375 size:1076140688 collection:"collection2" file_count:21880 delete_count:2 deleted_byte_count:51103 replica_placement:100 version:3 modified_at_second:1615016787 - volume id:378 size:959273824 collection:"collection2" file_count:15031 replica_placement:100 version:3 modified_at_second:1615632323 - volume id:379 size:1014108592 collection:"collection2" file_count:15360 delete_count:8 deleted_byte_count:2524591 replica_placement:100 version:3 modified_at_second:1615632323 - volume id:380 size:1010760464 collection:"collection2" file_count:14920 delete_count:7 deleted_byte_count:134859 replica_placement:100 version:3 modified_at_second:1615632322 - Disk hdd total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457 - DataNode 192.168.1.1:8080 total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457 - Rack DefaultRack total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457 - DataCenter dc4 total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457 - DataCenter dc5 hdd(volume:299/3000 active:299 free:2701 remote:0) - Rack DefaultRack hdd(volume:299/3000 active:299 free:2701 remote:0) - DataNode 192.168.1.5:8080 hdd(volume:299/3000 active:299 free:2701 remote:0) - Disk hdd(volume:299/3000 active:299 free:2701 remote:0) - volume id:5 size:293806008 file_count:1669 delete_count:2 deleted_byte_count:274334 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631342 - volume id:11 size:1109224552 collection:"collection0" file_count:44 replica_placement:100 version:3 modified_at_second:1615629606 - volume id:18 size:1096678688 collection:"collection0" file_count:88 delete_count:4 deleted_byte_count:8633 replica_placement:100 version:3 modified_at_second:1615631673 - volume id:19 size:1096923792 collection:"collection0" file_count:100 delete_count:10 deleted_byte_count:75779917 replica_placement:100 version:3 compact_revision:4 modified_at_second:1615630117 - volume id:20 size:1074760432 collection:"collection0" file_count:82 delete_count:5 deleted_byte_count:12156 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615629340 - volume id:26 size:532657632 collection:"collection0" file_count:100 delete_count:4 deleted_byte_count:9081 replica_placement:100 version:3 modified_at_second:1614170024 - volume id:27 size:298886792 file_count:1608 replica_placement:100 version:3 modified_at_second:1615632482 - volume id:28 size:308919192 file_count:1591 delete_count:1 deleted_byte_count:125280 replica_placement:100 version:3 modified_at_second:1615631762 - volume id:29 size:281582688 file_count:1537 replica_placement:100 version:3 modified_at_second:1615629422 - volume id:30 size:289466144 file_count:1566 delete_count:1 deleted_byte_count:124972 replica_placement:100 version:3 modified_at_second:1615632422 - volume id:31 size:273363256 file_count:1498 replica_placement:100 version:3 modified_at_second:1615631642 - volume id:32 size:281343360 file_count:1497 replica_placement:100 version:3 modified_at_second:1615632362 - volume id:33 size:1130226344 collection:"collection1" file_count:7322 delete_count:172 deleted_byte_count:45199399 replica_placement:100 version:3 modified_at_second:1615618789 - volume id:34 size:1077111136 collection:"collection1" file_count:9781 delete_count:110 deleted_byte_count:20894827 replica_placement:100 version:3 modified_at_second:1615619366 - volume id:35 size:1075241744 collection:"collection1" file_count:10523 delete_count:97 deleted_byte_count:46586217 replica_placement:100 version:3 modified_at_second:1615618790 - volume id:36 size:1075118336 collection:"collection1" file_count:10341 delete_count:118 deleted_byte_count:24753278 replica_placement:100 version:3 modified_at_second:1615606148 - volume id:37 size:1075895576 collection:"collection1" file_count:12013 delete_count:98 deleted_byte_count:50747932 replica_placement:100 version:3 modified_at_second:1615594776 - volume id:38 size:1075545744 collection:"collection1" file_count:13324 delete_count:100 deleted_byte_count:25223906 replica_placement:100 version:3 modified_at_second:1615569830 - volume id:39 size:1076606536 collection:"collection1" file_count:12612 delete_count:78 deleted_byte_count:17462730 replica_placement:100 version:3 modified_at_second:1615611959 - volume id:40 size:1075358552 collection:"collection1" file_count:12597 delete_count:62 deleted_byte_count:11657901 replica_placement:100 version:3 modified_at_second:1615612994 - volume id:41 size:1076283592 collection:"collection1" file_count:12088 delete_count:84 deleted_byte_count:19311237 replica_placement:100 version:3 modified_at_second:1615596736 - volume id:42 size:1093948352 collection:"collection1" file_count:7889 delete_count:47 deleted_byte_count:5697275 replica_placement:100 version:3 modified_at_second:1615548906 - volume id:43 size:1116445864 collection:"collection1" file_count:7355 delete_count:57 deleted_byte_count:9727158 replica_placement:100 version:3 modified_at_second:1615566167 - volume id:44 size:1077582560 collection:"collection1" file_count:7295 delete_count:50 deleted_byte_count:12618414 replica_placement:100 version:3 modified_at_second:1615566170 - volume id:45 size:1075254640 collection:"collection1" file_count:10772 delete_count:76 deleted_byte_count:22426345 replica_placement:100 version:3 modified_at_second:1615573498 - volume id:46 size:1075286056 collection:"collection1" file_count:9947 delete_count:309 deleted_byte_count:105601163 replica_placement:100 version:3 modified_at_second:1615569825 - volume id:47 size:444599784 collection:"collection1" file_count:709 delete_count:19 deleted_byte_count:11913451 replica_placement:100 version:3 modified_at_second:1615632397 - volume id:48 size:1076778664 collection:"collection1" file_count:9850 delete_count:77 deleted_byte_count:16641907 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630690 - volume id:49 size:1078775288 collection:"collection1" file_count:9631 delete_count:27 deleted_byte_count:5985628 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615575823 - volume id:50 size:1076688288 collection:"collection1" file_count:7921 delete_count:26 deleted_byte_count:5162032 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615610876 - volume id:51 size:1076796120 collection:"collection1" file_count:10550 delete_count:39 deleted_byte_count:12723654 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615547786 - volume id:53 size:1063089216 collection:"collection1" file_count:9832 delete_count:31 deleted_byte_count:9273066 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632156 - volume id:54 size:1045022288 collection:"collection1" file_count:9409 delete_count:29 deleted_byte_count:15102818 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630813 - volume id:55 size:1012890016 collection:"collection1" file_count:8651 delete_count:27 deleted_byte_count:9418841 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631453 - volume id:56 size:1002412240 collection:"collection1" file_count:8762 delete_count:40 deleted_byte_count:65885831 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632014 - volume id:57 size:839849792 collection:"collection1" file_count:7514 delete_count:24 deleted_byte_count:6228543 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631775 - volume id:58 size:908064192 collection:"collection1" file_count:8128 delete_count:21 deleted_byte_count:6113731 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632343 - volume id:59 size:988302272 collection:"collection1" file_count:8098 delete_count:20 deleted_byte_count:3947615 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632238 - volume id:60 size:1010702480 collection:"collection1" file_count:8969 delete_count:79 deleted_byte_count:24782814 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632439 - volume id:61 size:975604544 collection:"collection1" file_count:8683 delete_count:20 deleted_byte_count:10276072 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615631176 - volume id:62 size:873845904 collection:"collection1" file_count:7897 delete_count:23 deleted_byte_count:10920170 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631133 - volume id:63 size:956941176 collection:"collection1" file_count:8271 delete_count:32 deleted_byte_count:15876189 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632036 - volume id:64 size:965638424 collection:"collection1" file_count:8218 delete_count:27 deleted_byte_count:6922489 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631032 - volume id:65 size:823283608 collection:"collection1" file_count:7834 delete_count:29 deleted_byte_count:5950610 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632307 - volume id:66 size:821343440 collection:"collection1" file_count:7383 delete_count:29 deleted_byte_count:12010343 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631968 - volume id:67 size:878713880 collection:"collection1" file_count:7299 delete_count:117 deleted_byte_count:24857326 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632156 - volume id:69 size:863913896 collection:"collection1" file_count:7291 delete_count:100 deleted_byte_count:25335024 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630534 - volume id:70 size:886695472 collection:"collection1" file_count:7769 delete_count:164 deleted_byte_count:45162513 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632398 - volume id:71 size:907608392 collection:"collection1" file_count:7658 delete_count:122 deleted_byte_count:27622941 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632307 - volume id:72 size:903990720 collection:"collection1" file_count:6996 delete_count:240 deleted_byte_count:74147727 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630985 - volume id:73 size:929047544 collection:"collection1" file_count:7038 delete_count:227 deleted_byte_count:65336664 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630707 - volume id:75 size:908045000 collection:"collection1" file_count:6911 delete_count:268 deleted_byte_count:73934373 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632430 - volume id:76 size:985296744 collection:"collection1" file_count:6566 delete_count:61 deleted_byte_count:44464430 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632284 - volume id:77 size:929398296 collection:"collection1" file_count:7427 delete_count:238 deleted_byte_count:59581579 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632014 - volume id:78 size:1075671512 collection:"collection1" file_count:7540 delete_count:258 deleted_byte_count:71726846 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615582829 - volume id:79 size:948225472 collection:"collection1" file_count:6997 delete_count:227 deleted_byte_count:60625763 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631326 - volume id:80 size:827912928 collection:"collection1" file_count:6916 delete_count:15 deleted_byte_count:5611604 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631159 - volume id:81 size:880693168 collection:"collection1" file_count:7481 delete_count:238 deleted_byte_count:80880878 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631395 - volume id:82 size:1041660512 collection:"collection1" file_count:7043 delete_count:207 deleted_byte_count:52275724 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632430 - volume id:83 size:936194288 collection:"collection1" file_count:7593 delete_count:13 deleted_byte_count:4633917 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632029 - volume id:84 size:871262320 collection:"collection1" file_count:8190 delete_count:14 deleted_byte_count:3150948 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631161 - volume id:86 size:1009434632 collection:"collection1" file_count:8474 delete_count:236 deleted_byte_count:64543674 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630812 - volume id:87 size:922274624 collection:"collection1" file_count:12902 delete_count:13 deleted_byte_count:3412959 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632438 - volume id:88 size:1073767976 collection:"collection1" file_count:14994 delete_count:207 deleted_byte_count:82380696 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615541383 - volume id:89 size:1044421824 collection:"collection1" file_count:14943 delete_count:243 deleted_byte_count:58543159 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632208 - volume id:90 size:891163760 collection:"collection1" file_count:14608 delete_count:10 deleted_byte_count:2564369 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615629392 - volume id:91 size:936573952 collection:"collection1" file_count:14686 delete_count:11 deleted_byte_count:4717727 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631851 - volume id:92 size:992439144 collection:"collection1" file_count:7061 delete_count:195 deleted_byte_count:60649573 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630566 - volume id:93 size:1079602592 collection:"collection1" file_count:7878 delete_count:270 deleted_byte_count:74150048 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615556013 - volume id:94 size:1030684704 collection:"collection1" file_count:7660 delete_count:207 deleted_byte_count:70150733 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631616 - volume id:95 size:990877824 collection:"collection1" file_count:6620 delete_count:206 deleted_byte_count:60363604 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615631867 - volume id:96 size:989294848 collection:"collection1" file_count:7544 delete_count:229 deleted_byte_count:59931853 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630778 - volume id:98 size:1077836472 collection:"collection1" file_count:7605 delete_count:202 deleted_byte_count:73180379 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615523691 - volume id:99 size:1071718496 collection:"collection1" file_count:7470 delete_count:8 deleted_byte_count:9624950 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631175 - volume id:100 size:1083617472 collection:"collection1" file_count:7018 delete_count:187 deleted_byte_count:61304236 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615505914 - volume id:101 size:1077109408 collection:"collection1" file_count:7706 delete_count:226 deleted_byte_count:77864780 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630994 - volume id:102 size:1074359920 collection:"collection1" file_count:7338 delete_count:7 deleted_byte_count:6499151 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615626682 - volume id:103 size:1075863904 collection:"collection1" file_count:7184 delete_count:186 deleted_byte_count:58872238 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615628417 - volume id:106 size:1075458680 collection:"collection1" file_count:7182 delete_count:307 deleted_byte_count:69349053 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615598137 - volume id:107 size:1073811776 collection:"collection1" file_count:7436 delete_count:168 deleted_byte_count:57747428 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615293569 - volume id:108 size:1074648024 collection:"collection1" file_count:7472 delete_count:194 deleted_byte_count:70864699 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615593231 - volume id:109 size:1075254560 collection:"collection1" file_count:7556 delete_count:263 deleted_byte_count:55155265 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615502487 - volume id:110 size:1076575744 collection:"collection1" file_count:6996 delete_count:163 deleted_byte_count:52954032 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615590786 - volume id:111 size:1073826176 collection:"collection1" file_count:7355 delete_count:155 deleted_byte_count:50083578 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615593232 - volume id:112 size:1076392512 collection:"collection1" file_count:8291 delete_count:156 deleted_byte_count:74120183 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615569823 - volume id:113 size:1076709184 collection:"collection1" file_count:9355 delete_count:177 deleted_byte_count:59796765 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615569822 - volume id:114 size:1074762792 collection:"collection1" file_count:8802 delete_count:156 deleted_byte_count:38470055 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615591826 - volume id:115 size:1076192296 collection:"collection1" file_count:7690 delete_count:154 deleted_byte_count:32267193 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615285296 - volume id:117 size:1073917192 collection:"collection1" file_count:9520 delete_count:114 deleted_byte_count:21835126 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615573712 - volume id:118 size:1074064344 collection:"collection1" file_count:8738 delete_count:15 deleted_byte_count:3460697 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615516264 - volume id:120 size:1076115928 collection:"collection1" file_count:9639 delete_count:118 deleted_byte_count:33357871 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615482567 - volume id:121 size:1078803320 collection:"collection1" file_count:10113 delete_count:441 deleted_byte_count:94128627 replica_placement:100 version:3 modified_at_second:1615506626 - volume id:122 size:1076235312 collection:"collection1" file_count:9106 delete_count:252 deleted_byte_count:93041272 replica_placement:100 version:3 modified_at_second:1615585912 - volume id:123 size:1080491112 collection:"collection1" file_count:10623 delete_count:302 deleted_byte_count:83956998 replica_placement:100 version:3 modified_at_second:1615585916 - volume id:124 size:1074519360 collection:"collection1" file_count:9457 delete_count:286 deleted_byte_count:74752459 replica_placement:100 version:3 modified_at_second:1615585913 - volume id:125 size:1088687040 collection:"collection1" file_count:9518 delete_count:281 deleted_byte_count:76037905 replica_placement:100 version:3 modified_at_second:1615585913 - volume id:126 size:1073867408 collection:"collection1" file_count:9320 delete_count:278 deleted_byte_count:94547424 replica_placement:100 version:3 modified_at_second:1615585911 - volume id:127 size:1074907336 collection:"collection1" file_count:9900 delete_count:133 deleted_byte_count:48570820 replica_placement:100 version:3 modified_at_second:1615612990 - volume id:128 size:1074874632 collection:"collection1" file_count:9821 delete_count:148 deleted_byte_count:43633334 replica_placement:100 version:3 modified_at_second:1615602670 - volume id:129 size:1074704328 collection:"collection1" file_count:10012 delete_count:150 deleted_byte_count:64491721 replica_placement:100 version:3 modified_at_second:1615627566 - volume id:130 size:1075000632 collection:"collection1" file_count:10633 delete_count:161 deleted_byte_count:34768201 replica_placement:100 version:3 modified_at_second:1615582327 - volume id:131 size:1075279584 collection:"collection1" file_count:10075 delete_count:135 deleted_byte_count:29795712 replica_placement:100 version:3 modified_at_second:1615523898 - volume id:132 size:1088539552 collection:"collection1" file_count:11051 delete_count:71 deleted_byte_count:17178322 replica_placement:100 version:3 modified_at_second:1615619581 - volume id:134 size:1074367304 collection:"collection1" file_count:10662 delete_count:69 deleted_byte_count:25530139 replica_placement:100 version:3 modified_at_second:1615555873 - volume id:135 size:1073906776 collection:"collection1" file_count:10446 delete_count:71 deleted_byte_count:28599975 replica_placement:100 version:3 modified_at_second:1615569816 - volume id:136 size:1074433552 collection:"collection1" file_count:9593 delete_count:72 deleted_byte_count:26912512 replica_placement:100 version:3 modified_at_second:1615376036 - volume id:137 size:1074309264 collection:"collection1" file_count:9633 delete_count:50 deleted_byte_count:27487972 replica_placement:100 version:3 modified_at_second:1615572231 - volume id:138 size:1074465744 collection:"collection1" file_count:10120 delete_count:55 deleted_byte_count:15875438 replica_placement:100 version:3 modified_at_second:1615572231 - volume id:140 size:1076203744 collection:"collection1" file_count:11219 delete_count:57 deleted_byte_count:19864498 replica_placement:100 version:3 modified_at_second:1615571947 - volume id:141 size:1074619488 collection:"collection1" file_count:9840 delete_count:45 deleted_byte_count:40890181 replica_placement:100 version:3 modified_at_second:1615630994 - volume id:142 size:1075733064 collection:"collection1" file_count:9009 delete_count:48 deleted_byte_count:9912854 replica_placement:100 version:3 modified_at_second:1615598913 - volume id:143 size:1075011280 collection:"collection1" file_count:9608 delete_count:51 deleted_byte_count:37282460 replica_placement:100 version:3 modified_at_second:1615488584 - volume id:144 size:1074549720 collection:"collection1" file_count:8780 delete_count:50 deleted_byte_count:52475146 replica_placement:100 version:3 modified_at_second:1615573451 - volume id:145 size:1074394928 collection:"collection1" file_count:9255 delete_count:34 deleted_byte_count:38011392 replica_placement:100 version:3 modified_at_second:1615591825 - volume id:146 size:1076337576 collection:"collection1" file_count:10492 delete_count:50 deleted_byte_count:17071505 replica_placement:100 version:3 modified_at_second:1615632005 - volume id:147 size:1077130576 collection:"collection1" file_count:10451 delete_count:27 deleted_byte_count:8290907 replica_placement:100 version:3 modified_at_second:1615604115 - volume id:148 size:1076066568 collection:"collection1" file_count:9547 delete_count:33 deleted_byte_count:7034089 replica_placement:100 version:3 modified_at_second:1615586390 - volume id:149 size:1074989016 collection:"collection1" file_count:8352 delete_count:35 deleted_byte_count:7179742 replica_placement:100 version:3 modified_at_second:1615494494 - volume id:150 size:1076290328 collection:"collection1" file_count:9328 delete_count:33 deleted_byte_count:43417791 replica_placement:100 version:3 modified_at_second:1615611567 - volume id:152 size:1075941400 collection:"collection1" file_count:9951 delete_count:36 deleted_byte_count:25348335 replica_placement:100 version:3 modified_at_second:1615606614 - volume id:153 size:1078539784 collection:"collection1" file_count:10924 delete_count:34 deleted_byte_count:12603081 replica_placement:100 version:3 modified_at_second:1615606614 - volume id:154 size:1081244696 collection:"collection1" file_count:11002 delete_count:31 deleted_byte_count:8467560 replica_placement:100 version:3 modified_at_second:1615478469 - volume id:155 size:1075140688 collection:"collection1" file_count:10882 delete_count:32 deleted_byte_count:10076804 replica_placement:100 version:3 modified_at_second:1615606614 - volume id:156 size:1074975832 collection:"collection1" file_count:9535 delete_count:40 deleted_byte_count:11426621 replica_placement:100 version:3 modified_at_second:1615628341 - volume id:157 size:1076758536 collection:"collection1" file_count:10012 delete_count:19 deleted_byte_count:11688737 replica_placement:100 version:3 modified_at_second:1615597782 - volume id:158 size:1087251976 collection:"collection1" file_count:9972 delete_count:20 deleted_byte_count:10328429 replica_placement:100 version:3 modified_at_second:1615588879 - volume id:159 size:1074132368 collection:"collection1" file_count:9382 delete_count:27 deleted_byte_count:11474574 replica_placement:100 version:3 modified_at_second:1615593593 - volume id:160 size:1075680952 collection:"collection1" file_count:9772 delete_count:22 deleted_byte_count:4981968 replica_placement:100 version:3 modified_at_second:1615597780 - volume id:162 size:1074286880 collection:"collection1" file_count:11220 delete_count:17 deleted_byte_count:1815547 replica_placement:100 version:3 modified_at_second:1615478126 - volume id:163 size:1074457192 collection:"collection1" file_count:12524 delete_count:27 deleted_byte_count:6359619 replica_placement:100 version:3 modified_at_second:1615579313 - volume id:164 size:1074261248 collection:"collection1" file_count:11922 delete_count:25 deleted_byte_count:2923288 replica_placement:100 version:3 modified_at_second:1615620084 - volume id:165 size:1073891016 collection:"collection1" file_count:9152 delete_count:12 deleted_byte_count:19164659 replica_placement:100 version:3 modified_at_second:1615471907 - volume id:166 size:1075637536 collection:"collection1" file_count:14211 delete_count:24 deleted_byte_count:20415490 replica_placement:100 version:3 modified_at_second:1615491019 - volume id:168 size:1074718808 collection:"collection1" file_count:25702 delete_count:40 deleted_byte_count:4024775 replica_placement:100 version:3 modified_at_second:1615585664 - volume id:169 size:1073863128 collection:"collection1" file_count:25248 delete_count:43 deleted_byte_count:3013817 replica_placement:100 version:3 modified_at_second:1615569832 - volume id:170 size:1075747096 collection:"collection1" file_count:24596 delete_count:41 deleted_byte_count:3494711 replica_placement:100 version:3 modified_at_second:1615579204 - volume id:171 size:1081881312 collection:"collection1" file_count:24215 delete_count:36 deleted_byte_count:3191335 replica_placement:100 version:3 modified_at_second:1615596485 - volume id:172 size:1074787312 collection:"collection1" file_count:31236 delete_count:50 deleted_byte_count:3316482 replica_placement:100 version:3 modified_at_second:1615612385 - volume id:173 size:1074154648 collection:"collection1" file_count:30884 delete_count:34 deleted_byte_count:2430948 replica_placement:100 version:3 modified_at_second:1615591904 - volume id:175 size:1077742504 collection:"collection1" file_count:32353 delete_count:33 deleted_byte_count:1861403 replica_placement:100 version:3 modified_at_second:1615559515 - volume id:176 size:1073854800 collection:"collection1" file_count:30582 delete_count:34 deleted_byte_count:7701976 replica_placement:100 version:3 modified_at_second:1615626169 - volume id:177 size:1074120120 collection:"collection1" file_count:22293 delete_count:16 deleted_byte_count:3719562 replica_placement:100 version:3 modified_at_second:1615516891 - volume id:178 size:1087560112 collection:"collection1" file_count:23482 delete_count:22 deleted_byte_count:18810492 replica_placement:100 version:3 modified_at_second:1615541369 - volume id:180 size:1078438536 collection:"collection1" file_count:23614 delete_count:12 deleted_byte_count:4496474 replica_placement:100 version:3 modified_at_second:1614773242 - volume id:181 size:1074571768 collection:"collection1" file_count:22898 delete_count:19 deleted_byte_count:6628413 replica_placement:100 version:3 modified_at_second:1614745116 - volume id:182 size:1076131280 collection:"collection1" file_count:31987 delete_count:21 deleted_byte_count:1416142 replica_placement:100 version:3 modified_at_second:1615568922 - volume id:183 size:1076361448 collection:"collection1" file_count:31293 delete_count:16 deleted_byte_count:468841 replica_placement:100 version:3 modified_at_second:1615572982 - volume id:184 size:1074594160 collection:"collection1" file_count:31368 delete_count:22 deleted_byte_count:857453 replica_placement:100 version:3 modified_at_second:1615586578 - volume id:185 size:1074099624 collection:"collection1" file_count:30612 delete_count:17 deleted_byte_count:2610847 replica_placement:100 version:3 modified_at_second:1615506832 - volume id:186 size:1074220864 collection:"collection1" file_count:31450 delete_count:15 deleted_byte_count:391855 replica_placement:100 version:3 modified_at_second:1615614933 - volume id:187 size:1074395944 collection:"collection1" file_count:31853 delete_count:17 deleted_byte_count:454283 replica_placement:100 version:3 modified_at_second:1615590490 - volume id:188 size:1074732792 collection:"collection1" file_count:31867 delete_count:19 deleted_byte_count:393743 replica_placement:100 version:3 modified_at_second:1615487645 - volume id:189 size:1074847896 collection:"collection1" file_count:31450 delete_count:16 deleted_byte_count:1040552 replica_placement:100 version:3 modified_at_second:1615335661 - volume id:190 size:1074008912 collection:"collection1" file_count:31987 delete_count:11 deleted_byte_count:685125 replica_placement:100 version:3 modified_at_second:1615447161 - volume id:191 size:1075493024 collection:"collection1" file_count:31301 delete_count:19 deleted_byte_count:708401 replica_placement:100 version:3 modified_at_second:1615357456 - volume id:192 size:1075857400 collection:"collection1" file_count:31490 delete_count:25 deleted_byte_count:720617 replica_placement:100 version:3 modified_at_second:1615621632 - volume id:193 size:1076616768 collection:"collection1" file_count:31907 delete_count:16 deleted_byte_count:464900 replica_placement:100 version:3 modified_at_second:1615507875 - volume id:194 size:1073985624 collection:"collection1" file_count:31434 delete_count:18 deleted_byte_count:391432 replica_placement:100 version:3 modified_at_second:1615559499 - volume id:195 size:1074158312 collection:"collection1" file_count:31453 delete_count:15 deleted_byte_count:718266 replica_placement:100 version:3 modified_at_second:1615559331 - volume id:196 size:1074594784 collection:"collection1" file_count:31665 delete_count:18 deleted_byte_count:3468922 replica_placement:100 version:3 modified_at_second:1615501688 - volume id:197 size:1075423296 collection:"collection1" file_count:16473 delete_count:15 deleted_byte_count:12552442 replica_placement:100 version:3 modified_at_second:1615485253 - volume id:198 size:1075104712 collection:"collection1" file_count:16577 delete_count:18 deleted_byte_count:6583181 replica_placement:100 version:3 modified_at_second:1615623369 - volume id:199 size:1078117688 collection:"collection1" file_count:16497 delete_count:14 deleted_byte_count:1514286 replica_placement:100 version:3 modified_at_second:1615585984 - volume id:200 size:1075630536 collection:"collection1" file_count:16380 delete_count:18 deleted_byte_count:1103109 replica_placement:100 version:3 modified_at_second:1615485252 - volume id:201 size:1091460440 collection:"collection1" file_count:16684 delete_count:26 deleted_byte_count:5590335 replica_placement:100 version:3 modified_at_second:1615585987 - volume id:202 size:1077533160 collection:"collection1" file_count:2847 delete_count:67 deleted_byte_count:65172985 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615588497 - volume id:203 size:1027316272 collection:"collection1" file_count:3040 delete_count:11 deleted_byte_count:3993230 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631728 - volume id:204 size:1079766872 collection:"collection1" file_count:3233 delete_count:255 deleted_byte_count:104707641 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615565701 - volume id:205 size:1078485304 collection:"collection1" file_count:2869 delete_count:43 deleted_byte_count:18290259 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615579314 - volume id:206 size:1082045848 collection:"collection1" file_count:2979 delete_count:225 deleted_byte_count:88220074 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630989 - volume id:207 size:1081939960 collection:"collection1" file_count:3010 delete_count:4 deleted_byte_count:692350 replica_placement:100 version:3 modified_at_second:1615269061 - volume id:208 size:1077863624 collection:"collection1" file_count:3147 delete_count:6 deleted_byte_count:858726 replica_placement:100 version:3 modified_at_second:1615495515 - volume id:210 size:1094311304 collection:"collection1" file_count:3468 delete_count:4 deleted_byte_count:466433 replica_placement:100 version:3 modified_at_second:1615495515 - volume id:212 size:1078293448 collection:"collection1" file_count:3106 delete_count:6 deleted_byte_count:2085755 replica_placement:100 version:3 modified_at_second:1615586387 - volume id:213 size:1093588072 collection:"collection1" file_count:3681 delete_count:12 deleted_byte_count:3138791 replica_placement:100 version:3 modified_at_second:1615586387 - volume id:214 size:1074486992 collection:"collection1" file_count:3217 delete_count:10 deleted_byte_count:6392871 replica_placement:100 version:3 modified_at_second:1615586383 - volume id:215 size:1074798704 collection:"collection1" file_count:2819 delete_count:31 deleted_byte_count:10873569 replica_placement:100 version:3 modified_at_second:1615586386 - volume id:217 size:1075381872 collection:"collection1" file_count:3331 delete_count:14 deleted_byte_count:2009141 replica_placement:100 version:3 modified_at_second:1615401638 - volume id:218 size:1081263944 collection:"collection1" file_count:3433 delete_count:14 deleted_byte_count:3454237 replica_placement:100 version:3 modified_at_second:1615603637 - volume id:219 size:1092298816 collection:"collection1" file_count:3193 delete_count:17 deleted_byte_count:2047576 replica_placement:100 version:3 modified_at_second:1615579316 - volume id:220 size:1081928312 collection:"collection1" file_count:3166 delete_count:13 deleted_byte_count:4127709 replica_placement:100 version:3 modified_at_second:1615579317 - volume id:221 size:1106545456 collection:"collection1" file_count:3153 delete_count:11 deleted_byte_count:1496835 replica_placement:100 version:3 modified_at_second:1615269138 - volume id:222 size:1106623104 collection:"collection1" file_count:3273 delete_count:11 deleted_byte_count:2114627 replica_placement:100 version:3 modified_at_second:1615586243 - volume id:223 size:1075233064 collection:"collection1" file_count:2966 delete_count:9 deleted_byte_count:744001 replica_placement:100 version:3 modified_at_second:1615586244 - volume id:224 size:1093691520 collection:"collection1" file_count:3463 delete_count:10 deleted_byte_count:1128328 replica_placement:100 version:3 modified_at_second:1615601870 - volume id:225 size:1080698928 collection:"collection1" file_count:3115 delete_count:7 deleted_byte_count:18170416 replica_placement:100 version:3 modified_at_second:1615434684 - volume id:226 size:1103504768 collection:"collection1" file_count:2965 delete_count:10 deleted_byte_count:2639254 replica_placement:100 version:3 modified_at_second:1615601867 - volume id:228 size:1109784072 collection:"collection1" file_count:2504 delete_count:24 deleted_byte_count:5458950 replica_placement:100 version:3 modified_at_second:1615610489 - volume id:230 size:1080722984 collection:"collection1" file_count:2898 delete_count:15 deleted_byte_count:3929261 replica_placement:100 version:3 modified_at_second:1615610537 - volume id:232 size:1073901520 collection:"collection1" file_count:3004 delete_count:54 deleted_byte_count:10273081 replica_placement:100 version:3 modified_at_second:1615611351 - volume id:234 size:1073835280 collection:"collection1" file_count:2965 delete_count:41 deleted_byte_count:4960354 replica_placement:100 version:3 modified_at_second:1615611351 - volume id:235 size:1075586104 collection:"collection1" file_count:2767 delete_count:33 deleted_byte_count:3216540 replica_placement:100 version:3 modified_at_second:1615611354 - volume id:236 size:1089476136 collection:"collection1" file_count:3231 delete_count:53 deleted_byte_count:11625921 replica_placement:100 version:3 modified_at_second:1615611351 - volume id:237 size:375722792 collection:"collection1" file_count:736 delete_count:16 deleted_byte_count:4464870 replica_placement:100 version:3 modified_at_second:1615631727 - volume id:238 size:354320000 collection:"collection1" file_count:701 delete_count:17 deleted_byte_count:5940420 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632030 - volume id:239 size:426569024 collection:"collection1" file_count:693 delete_count:19 deleted_byte_count:13020783 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630841 - volume id:240 size:424791528 collection:"collection1" file_count:733 delete_count:13 deleted_byte_count:7515220 replica_placement:100 version:3 modified_at_second:1615631670 - volume id:241 size:380217424 collection:"collection1" file_count:633 delete_count:6 deleted_byte_count:1715768 replica_placement:100 version:3 modified_at_second:1615632006 - volume id:242 size:1075383392 collection:"collection2" file_count:10470 replica_placement:100 version:3 modified_at_second:1614852116 - volume id:243 size:1088174704 collection:"collection2" file_count:11109 delete_count:1 deleted_byte_count:938 replica_placement:100 version:3 modified_at_second:1614852203 - volume id:244 size:1080295352 collection:"collection2" file_count:10812 delete_count:1 deleted_byte_count:795 replica_placement:100 version:3 modified_at_second:1615628825 - volume id:246 size:1075998672 collection:"collection2" file_count:10365 delete_count:1 deleted_byte_count:13112 replica_placement:100 version:3 modified_at_second:1614852106 - volume id:247 size:1075859808 collection:"collection2" file_count:10443 delete_count:2 deleted_byte_count:564486 replica_placement:100 version:3 modified_at_second:1614856152 - volume id:248 size:1084301208 collection:"collection2" file_count:11217 delete_count:4 deleted_byte_count:746488 replica_placement:100 version:3 modified_at_second:1614856285 - volume id:250 size:1080572168 collection:"collection2" file_count:10220 replica_placement:100 version:3 modified_at_second:1614856129 - volume id:252 size:1075065264 collection:"collection2" file_count:14622 delete_count:2 deleted_byte_count:5228 replica_placement:100 version:3 modified_at_second:1614861200 - volume id:253 size:1087328880 collection:"collection2" file_count:14920 delete_count:3 deleted_byte_count:522994 replica_placement:100 version:3 modified_at_second:1614861258 - volume id:254 size:1074830736 collection:"collection2" file_count:14140 delete_count:2 deleted_byte_count:105892 replica_placement:100 version:3 modified_at_second:1614861115 - volume id:255 size:1079581640 collection:"collection2" file_count:14877 delete_count:3 deleted_byte_count:101223 replica_placement:100 version:3 modified_at_second:1614861233 - volume id:256 size:1074283592 collection:"collection2" file_count:14157 delete_count:1 deleted_byte_count:18156 replica_placement:100 version:3 modified_at_second:1614861100 - volume id:257 size:1082621720 collection:"collection2" file_count:18172 delete_count:2 deleted_byte_count:25125 replica_placement:100 version:3 modified_at_second:1614866402 - volume id:258 size:1075527216 collection:"collection2" file_count:18421 delete_count:4 deleted_byte_count:267833 replica_placement:100 version:3 modified_at_second:1614866420 - volume id:259 size:1075507848 collection:"collection2" file_count:18079 delete_count:2 deleted_byte_count:71992 replica_placement:100 version:3 modified_at_second:1614866381 - volume id:260 size:1075105664 collection:"collection2" file_count:17316 delete_count:4 deleted_byte_count:2015310 replica_placement:100 version:3 modified_at_second:1614866226 - volume id:261 size:1076628592 collection:"collection2" file_count:18355 delete_count:1 deleted_byte_count:1155 replica_placement:100 version:3 modified_at_second:1614866420 - volume id:262 size:1078492584 collection:"collection2" file_count:20390 delete_count:3 deleted_byte_count:287601 replica_placement:100 version:3 modified_at_second:1614871601 - volume id:264 size:1081624192 collection:"collection2" file_count:21151 replica_placement:100 version:3 modified_at_second:1614871629 - volume id:265 size:1076401104 collection:"collection2" file_count:19932 delete_count:2 deleted_byte_count:160823 replica_placement:100 version:3 modified_at_second:1614871543 - volume id:266 size:1075617552 collection:"collection2" file_count:20075 delete_count:1 deleted_byte_count:1039 replica_placement:100 version:3 modified_at_second:1614871526 - volume id:267 size:1075699376 collection:"collection2" file_count:21039 delete_count:3 deleted_byte_count:59956 replica_placement:100 version:3 modified_at_second:1614877294 - volume id:270 size:1076876424 collection:"collection2" file_count:22057 delete_count:1 deleted_byte_count:43916 replica_placement:100 version:3 modified_at_second:1614877469 - volume id:271 size:1076992704 collection:"collection2" file_count:22640 delete_count:1 deleted_byte_count:30645 replica_placement:100 version:3 modified_at_second:1614877504 - volume id:272 size:1076145912 collection:"collection2" file_count:21034 delete_count:2 deleted_byte_count:216564 replica_placement:100 version:3 modified_at_second:1614884139 - volume id:273 size:1074873432 collection:"collection2" file_count:20511 delete_count:3 deleted_byte_count:46076 replica_placement:100 version:3 modified_at_second:1614884046 - volume id:274 size:1075994184 collection:"collection2" file_count:20997 replica_placement:100 version:3 modified_at_second:1614884113 - volume id:275 size:1078349024 collection:"collection2" file_count:20808 delete_count:1 deleted_byte_count:1118 replica_placement:100 version:3 modified_at_second:1614884147 - volume id:276 size:1076899880 collection:"collection2" file_count:20190 delete_count:1 deleted_byte_count:8798 replica_placement:100 version:3 modified_at_second:1614884003 - volume id:278 size:1078798632 collection:"collection2" file_count:20597 delete_count:5 deleted_byte_count:400060 replica_placement:100 version:3 modified_at_second:1614890292 - volume id:280 size:1077432160 collection:"collection2" file_count:20286 delete_count:1 deleted_byte_count:879 replica_placement:100 version:3 modified_at_second:1614890262 - volume id:281 size:1077581064 collection:"collection2" file_count:20206 delete_count:3 deleted_byte_count:143964 replica_placement:100 version:3 modified_at_second:1614890237 - volume id:282 size:1075232184 collection:"collection2" file_count:22659 delete_count:4 deleted_byte_count:67915 replica_placement:100 version:3 modified_at_second:1614897304 - volume id:283 size:1080178880 collection:"collection2" file_count:19462 delete_count:7 deleted_byte_count:660407 replica_placement:100 version:3 modified_at_second:1614896623 - volume id:286 size:1077464816 collection:"collection2" file_count:23905 delete_count:6 deleted_byte_count:630577 replica_placement:100 version:3 modified_at_second:1614897401 - volume id:287 size:1074590536 collection:"collection2" file_count:28163 delete_count:5 deleted_byte_count:35727 replica_placement:100 version:3 modified_at_second:1614904875 - volume id:288 size:1075406920 collection:"collection2" file_count:27243 delete_count:2 deleted_byte_count:51519 replica_placement:100 version:3 modified_at_second:1614904738 - volume id:289 size:1075284312 collection:"collection2" file_count:29342 delete_count:5 deleted_byte_count:100454 replica_placement:100 version:3 modified_at_second:1614904977 - volume id:290 size:1074723800 collection:"collection2" file_count:28340 delete_count:4 deleted_byte_count:199064 replica_placement:100 version:3 modified_at_second:1614904924 - volume id:292 size:1092010672 collection:"collection2" file_count:26781 delete_count:5 deleted_byte_count:508910 replica_placement:100 version:3 modified_at_second:1614912325 - volume id:295 size:1074702320 collection:"collection2" file_count:24488 delete_count:3 deleted_byte_count:48555 replica_placement:100 version:3 modified_at_second:1614911929 - volume id:296 size:1077824056 collection:"collection2" file_count:26741 delete_count:4 deleted_byte_count:199906 replica_placement:100 version:3 modified_at_second:1614912301 - volume id:297 size:1080229176 collection:"collection2" file_count:23409 delete_count:5 deleted_byte_count:46268 replica_placement:100 version:3 modified_at_second:1614918481 - volume id:298 size:1075410024 collection:"collection2" file_count:23222 delete_count:2 deleted_byte_count:46110 replica_placement:100 version:3 modified_at_second:1614918474 - volume id:302 size:1077559640 collection:"collection2" file_count:23124 delete_count:7 deleted_byte_count:293111 replica_placement:100 version:3 modified_at_second:1614925500 - volume id:304 size:1081038944 collection:"collection2" file_count:24505 delete_count:2 deleted_byte_count:124447 replica_placement:100 version:3 modified_at_second:1614925569 - volume id:305 size:1074185376 collection:"collection2" file_count:22074 delete_count:5 deleted_byte_count:20221 replica_placement:100 version:3 modified_at_second:1614925312 - volume id:306 size:1074763952 collection:"collection2" file_count:22939 replica_placement:100 version:3 modified_at_second:1614925462 - volume id:307 size:1076567912 collection:"collection2" file_count:23377 delete_count:2 deleted_byte_count:25453 replica_placement:100 version:3 modified_at_second:1614931448 - volume id:308 size:1074022336 collection:"collection2" file_count:23086 delete_count:2 deleted_byte_count:2127 replica_placement:100 version:3 modified_at_second:1614931401 - volume id:311 size:1088248344 collection:"collection2" file_count:23553 delete_count:6 deleted_byte_count:191716 replica_placement:100 version:3 modified_at_second:1614931463 - volume id:312 size:1075037528 collection:"collection2" file_count:22524 replica_placement:100 version:3 modified_at_second:1614937831 - volume id:313 size:1074875960 collection:"collection2" file_count:22404 delete_count:4 deleted_byte_count:51728 replica_placement:100 version:3 modified_at_second:1614937755 - volume id:316 size:1077720776 collection:"collection2" file_count:22605 delete_count:1 deleted_byte_count:8503 replica_placement:100 version:3 modified_at_second:1614937838 - volume id:318 size:1075965168 collection:"collection2" file_count:22459 delete_count:2 deleted_byte_count:37778 replica_placement:100 version:3 modified_at_second:1614943862 - volume id:322 size:1078471536 collection:"collection2" file_count:21905 delete_count:3 deleted_byte_count:145002 replica_placement:100 version:3 modified_at_second:1614950572 - volume id:323 size:1074608056 collection:"collection2" file_count:21605 delete_count:4 deleted_byte_count:172090 replica_placement:100 version:3 modified_at_second:1614950526 - volume id:325 size:1080701232 collection:"collection2" file_count:21735 replica_placement:100 version:3 modified_at_second:1614950525 - volume id:326 size:1076059920 collection:"collection2" file_count:22564 delete_count:2 deleted_byte_count:192886 replica_placement:100 version:3 modified_at_second:1614950619 - volume id:327 size:1076121304 collection:"collection2" file_count:22007 delete_count:3 deleted_byte_count:60358 replica_placement:100 version:3 modified_at_second:1614956487 - volume id:328 size:1074767816 collection:"collection2" file_count:21720 delete_count:3 deleted_byte_count:56429 replica_placement:100 version:3 modified_at_second:1614956362 - volume id:329 size:1076691960 collection:"collection2" file_count:22411 delete_count:5 deleted_byte_count:214092 replica_placement:100 version:3 modified_at_second:1614956485 - volume id:330 size:1080825760 collection:"collection2" file_count:22464 delete_count:2 deleted_byte_count:15771 replica_placement:100 version:3 modified_at_second:1614956476 - volume id:331 size:1074957256 collection:"collection2" file_count:21230 delete_count:4 deleted_byte_count:62145 replica_placement:100 version:3 modified_at_second:1614956259 - volume id:332 size:1075569928 collection:"collection2" file_count:22097 delete_count:3 deleted_byte_count:98273 replica_placement:100 version:3 modified_at_second:1614962869 - volume id:333 size:1074270160 collection:"collection2" file_count:21271 delete_count:2 deleted_byte_count:168122 replica_placement:100 version:3 modified_at_second:1614962697 - volume id:334 size:1075607880 collection:"collection2" file_count:22546 delete_count:6 deleted_byte_count:101538 replica_placement:100 version:3 modified_at_second:1614962978 - volume id:335 size:1076235136 collection:"collection2" file_count:22391 delete_count:3 deleted_byte_count:8838 replica_placement:100 version:3 modified_at_second:1614962970 - volume id:337 size:1075646896 collection:"collection2" file_count:21934 delete_count:1 deleted_byte_count:3397 replica_placement:100 version:3 modified_at_second:1614969937 - volume id:339 size:1078402392 collection:"collection2" file_count:22309 replica_placement:100 version:3 modified_at_second:1614969995 - volume id:340 size:1079462152 collection:"collection2" file_count:22319 delete_count:4 deleted_byte_count:93620 replica_placement:100 version:3 modified_at_second:1614969977 - volume id:341 size:1074448360 collection:"collection2" file_count:21590 delete_count:5 deleted_byte_count:160085 replica_placement:100 version:3 modified_at_second:1614969858 - volume id:343 size:1075345072 collection:"collection2" file_count:21095 delete_count:2 deleted_byte_count:20581 replica_placement:100 version:3 modified_at_second:1614977148 - volume id:346 size:1076464112 collection:"collection2" file_count:22320 delete_count:4 deleted_byte_count:798258 replica_placement:100 version:3 modified_at_second:1614977511 - volume id:347 size:1075145248 collection:"collection2" file_count:22178 delete_count:1 deleted_byte_count:79392 replica_placement:100 version:3 modified_at_second:1614984727 - volume id:348 size:1080623544 collection:"collection2" file_count:21667 delete_count:1 deleted_byte_count:2443 replica_placement:100 version:3 modified_at_second:1614984604 - volume id:349 size:1075957672 collection:"collection2" file_count:22395 delete_count:2 deleted_byte_count:61565 replica_placement:100 version:3 modified_at_second:1614984748 - volume id:351 size:1078795120 collection:"collection2" file_count:23660 delete_count:3 deleted_byte_count:102141 replica_placement:100 version:3 modified_at_second:1614984816 - volume id:352 size:1077145936 collection:"collection2" file_count:22066 delete_count:1 deleted_byte_count:1018 replica_placement:100 version:3 modified_at_second:1614992130 - volume id:353 size:1074897496 collection:"collection2" file_count:21266 delete_count:2 deleted_byte_count:3105374 replica_placement:100 version:3 modified_at_second:1614991951 - volume id:354 size:1085214104 collection:"collection2" file_count:23150 delete_count:4 deleted_byte_count:82391 replica_placement:100 version:3 modified_at_second:1614992208 - volume id:357 size:1074276152 collection:"collection2" file_count:23137 delete_count:4 deleted_byte_count:188487 replica_placement:100 version:3 modified_at_second:1614998792 - volume id:359 size:1074211296 collection:"collection2" file_count:22437 delete_count:2 deleted_byte_count:187953 replica_placement:100 version:3 modified_at_second:1614998711 - volume id:360 size:1075532512 collection:"collection2" file_count:22574 delete_count:3 deleted_byte_count:1774776 replica_placement:100 version:3 modified_at_second:1614998770 - volume id:361 size:1075362744 collection:"collection2" file_count:22272 delete_count:1 deleted_byte_count:3497 replica_placement:100 version:3 modified_at_second:1614998668 - volume id:362 size:1074074176 collection:"collection2" file_count:20595 delete_count:1 deleted_byte_count:112145 replica_placement:100 version:3 modified_at_second:1615004407 - volume id:363 size:1078859640 collection:"collection2" file_count:23177 delete_count:4 deleted_byte_count:9601 replica_placement:100 version:3 modified_at_second:1615004823 - volume id:364 size:1081280880 collection:"collection2" file_count:22686 delete_count:1 deleted_byte_count:84375 replica_placement:100 version:3 modified_at_second:1615004813 - volume id:365 size:1075736632 collection:"collection2" file_count:22193 delete_count:5 deleted_byte_count:259033 replica_placement:100 version:3 modified_at_second:1615004776 - volume id:366 size:1075267272 collection:"collection2" file_count:21856 delete_count:5 deleted_byte_count:138363 replica_placement:100 version:3 modified_at_second:1615004703 - volume id:367 size:1076403648 collection:"collection2" file_count:22995 delete_count:2 deleted_byte_count:36955 replica_placement:100 version:3 modified_at_second:1615010985 - volume id:368 size:1074821960 collection:"collection2" file_count:22252 delete_count:4 deleted_byte_count:3291946 replica_placement:100 version:3 modified_at_second:1615010877 - volume id:369 size:1091472040 collection:"collection2" file_count:23709 delete_count:4 deleted_byte_count:400876 replica_placement:100 version:3 modified_at_second:1615011021 - volume id:370 size:1076040544 collection:"collection2" file_count:22092 delete_count:2 deleted_byte_count:115388 replica_placement:100 version:3 modified_at_second:1615010877 - volume id:371 size:1078806216 collection:"collection2" file_count:22685 delete_count:2 deleted_byte_count:68905 replica_placement:100 version:3 modified_at_second:1615010995 - volume id:372 size:1076193344 collection:"collection2" file_count:22774 delete_count:1 deleted_byte_count:3495 replica_placement:100 version:3 modified_at_second:1615016911 - volume id:373 size:1080928088 collection:"collection2" file_count:22617 delete_count:4 deleted_byte_count:91849 replica_placement:100 version:3 modified_at_second:1615016878 - volume id:374 size:1085011176 collection:"collection2" file_count:23054 delete_count:2 deleted_byte_count:89034 replica_placement:100 version:3 modified_at_second:1615016917 - volume id:376 size:1074845832 collection:"collection2" file_count:22908 delete_count:4 deleted_byte_count:432305 replica_placement:100 version:3 modified_at_second:1615016916 - volume id:377 size:957434264 collection:"collection2" file_count:14929 delete_count:1 deleted_byte_count:43099 replica_placement:100 version:3 modified_at_second:1615632323 - volume id:379 size:1014108528 collection:"collection2" file_count:15362 delete_count:6 deleted_byte_count:2481613 replica_placement:100 version:3 modified_at_second:1615632323 - Disk hdd total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660 - DataNode 192.168.1.5:8080 total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660 - Rack DefaultRack total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660 - DataCenter dc5 total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660 -total size:775256653592 file_count:10478712 deleted_file:33754 deleted_bytes:10839266043 -` +//go:embed sample.topo.txt +var topoData string diff --git a/weed/shell/command_volume_server_evacuate.go b/weed/shell/command_volume_server_evacuate.go index 6a8aeab52..31ebcfec1 100644 --- a/weed/shell/command_volume_server_evacuate.go +++ b/weed/shell/command_volume_server_evacuate.go @@ -78,7 +78,7 @@ func volumeServerEvacuate(commandEnv *CommandEnv, volumeServer string, skipNonMo // list all the volumes // collect topology information - topologyInfo, _, err := collectTopologyInfo(commandEnv) + topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) if err != nil { return err } diff --git a/weed/shell/command_volume_tier_download.go b/weed/shell/command_volume_tier_download.go index a2330ab8a..424a64b09 100644 --- a/weed/shell/command_volume_tier_download.go +++ b/weed/shell/command_volume_tier_download.go @@ -57,7 +57,7 @@ func (c *commandVolumeTierDownload) Do(args []string, commandEnv *CommandEnv, wr vid := needle.VolumeId(*volumeId) // collect topology information - topologyInfo, _, err := collectTopologyInfo(commandEnv) + topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) if err != nil { return err } diff --git a/weed/shell/command_volume_tier_move.go b/weed/shell/command_volume_tier_move.go index ff0a1a180..42cd3cf44 100644 --- a/weed/shell/command_volume_tier_move.go +++ b/weed/shell/command_volume_tier_move.go @@ -74,7 +74,7 @@ func (c *commandVolumeTierMove) Do(args []string, commandEnv *CommandEnv, writer } // collect topology information - topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv) + topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv, 0) if err != nil { return err } @@ -116,8 +116,10 @@ func (c *commandVolumeTierMove) Do(args []string, commandEnv *CommandEnv, writer unlock := c.Lock(job.src) - if err := c.doMoveOneVolume(commandEnv, writer, job.vid, toDiskType, locations, job.src, dst, applyChanges); err != nil { - fmt.Fprintf(writer, "move volume %d %s => %s: %v\n", job.vid, job.src, dst.dataNode.Id, err) + if applyChanges { + if err := c.doMoveOneVolume(commandEnv, writer, job.vid, toDiskType, locations, job.src, dst); err != nil { + fmt.Fprintf(writer, "move volume %d %s => %s: %v\n", job.vid, job.src, dst.dataNode.Id, err) + } } unlock() } @@ -216,34 +218,27 @@ func (c *commandVolumeTierMove) doVolumeTierMove(commandEnv *CommandEnv, writer return nil } -func (c *commandVolumeTierMove) doMoveOneVolume(commandEnv *CommandEnv, writer io.Writer, vid needle.VolumeId, toDiskType types.DiskType, locations []wdclient.Location, sourceVolumeServer pb.ServerAddress, dst location, applyChanges bool) (err error) { +func (c *commandVolumeTierMove) doMoveOneVolume(commandEnv *CommandEnv, writer io.Writer, vid needle.VolumeId, toDiskType types.DiskType, locations []wdclient.Location, sourceVolumeServer pb.ServerAddress, dst location) (err error) { // mark all replicas as read only - if applyChanges { - if err = markVolumeReplicasWritable(commandEnv.option.GrpcDialOption, vid, locations, false); err != nil { - return fmt.Errorf("mark volume %d as readonly on %s: %v", vid, locations[0].Url, err) - } - if err = LiveMoveVolume(commandEnv.option.GrpcDialOption, writer, vid, sourceVolumeServer, pb.NewServerAddressFromDataNode(dst.dataNode), 5*time.Second, toDiskType.ReadableString(), true); err != nil { - - // mark all replicas as writable - if err = markVolumeReplicasWritable(commandEnv.option.GrpcDialOption, vid, locations, true); err != nil { - glog.Errorf("mark volume %d as writable on %s: %v", vid, locations[0].Url, err) - } + if err = markVolumeReplicasWritable(commandEnv.option.GrpcDialOption, vid, locations, false); err != nil { + return fmt.Errorf("mark volume %d as readonly on %s: %v", vid, locations[0].Url, err) + } + if err = LiveMoveVolume(commandEnv.option.GrpcDialOption, writer, vid, sourceVolumeServer, pb.NewServerAddressFromDataNode(dst.dataNode), 5*time.Second, toDiskType.ReadableString(), true); err != nil { - return fmt.Errorf("move volume %d %s => %s : %v", vid, locations[0].Url, dst.dataNode.Id, err) + // mark all replicas as writable + if err = markVolumeReplicasWritable(commandEnv.option.GrpcDialOption, vid, locations, true); err != nil { + glog.Errorf("mark volume %d as writable on %s: %v", vid, locations[0].Url, err) } - } - // adjust volume count - dst.dataNode.DiskInfos[string(toDiskType)].VolumeCount++ + return fmt.Errorf("move volume %d %s => %s : %v", vid, locations[0].Url, dst.dataNode.Id, err) + } // remove the remaining replicas for _, loc := range locations { if loc.Url != dst.dataNode.Id && loc.ServerAddress() != sourceVolumeServer { - if applyChanges { - if err = deleteVolume(commandEnv.option.GrpcDialOption, vid, loc.ServerAddress()); err != nil { - fmt.Fprintf(writer, "failed to delete volume %d on %s: %v\n", vid, loc.Url, err) - } + if err = deleteVolume(commandEnv.option.GrpcDialOption, vid, loc.ServerAddress()); err != nil { + fmt.Fprintf(writer, "failed to delete volume %d on %s: %v\n", vid, loc.Url, err) } // reduce volume count? Not really necessary since they are "more" full and will not be a candidate to move to } diff --git a/weed/shell/sample.topo.txt b/weed/shell/sample.topo.txt new file mode 100644 index 000000000..b3ceaa140 --- /dev/null +++ b/weed/shell/sample.topo.txt @@ -0,0 +1,804 @@ +Topology volumeSizeLimit:1024 MB hdd(volume:760/7280 active:760 free:6520 remote:0) + DataCenter dc1 hdd(volume:0/0 active:0 free:0 remote:0) + Rack DefaultRack hdd(volume:0/0 active:0 free:0 remote:0) + Rack DefaultRack total size:0 file_count:0 + DataCenter dc1 total size:0 file_count:0 + DataCenter dc2 hdd(volume:86/430 active:86 free:344 remote:0) + Rack rack1 hdd(volume:50/240 active:50 free:190 remote:0) + DataNode 192.168.1.4:8080 hdd(volume:50/240 active:50 free:190 remote:0) + Disk hdd(volume:50/240 active:50 free:190 remote:0) + volume id:15 size:1115965064 collection:"collection0" file_count:83 replica_placement:100 version:3 modified_at_second:1609923671 + volume id:21 size:1097631536 collection:"collection0" file_count:82 delete_count:7 deleted_byte_count:68975485 replica_placement:100 version:3 modified_at_second:1609929578 + volume id:22 size:1086828272 collection:"collection0" file_count:75 replica_placement:100 version:3 modified_at_second:1609930001 + volume id:23 size:1076380216 collection:"collection0" file_count:68 replica_placement:100 version:3 modified_at_second:1609930434 + volume id:24 size:1074139776 collection:"collection0" file_count:90 replica_placement:100 version:3 modified_at_second:1609930909 + volume id:25 size:690757512 collection:"collection0" file_count:38 replica_placement:100 version:3 modified_at_second:1611144216 + volume id:27 size:298886792 file_count:1608 replica_placement:100 version:3 modified_at_second:1615632482 + volume id:28 size:308919192 file_count:1591 delete_count:1 deleted_byte_count:125280 replica_placement:100 version:3 modified_at_second:1615631762 + volume id:29 size:281582680 file_count:1537 replica_placement:100 version:3 modified_at_second:1615629422 + volume id:30 size:289466144 file_count:1566 delete_count:1 deleted_byte_count:124972 replica_placement:100 version:3 modified_at_second:1615632422 + volume id:31 size:273363256 file_count:1498 replica_placement:100 version:3 modified_at_second:1615631642 + volume id:33 size:1130226400 collection:"collection1" file_count:7322 delete_count:172 deleted_byte_count:45199399 replica_placement:100 version:3 modified_at_second:1615618789 + volume id:38 size:1075545744 collection:"collection1" file_count:13324 delete_count:100 deleted_byte_count:25223906 replica_placement:100 version:3 modified_at_second:1615569830 + volume id:51 size:1076796120 collection:"collection1" file_count:10550 delete_count:39 deleted_byte_count:12723654 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615547786 + volume id:52 size:1083529728 collection:"collection1" file_count:10128 delete_count:32 deleted_byte_count:10608391 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615599195 + volume id:54 size:1045022344 collection:"collection1" file_count:9408 delete_count:30 deleted_byte_count:15132106 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630812 + volume id:63 size:956941112 collection:"collection1" file_count:8271 delete_count:32 deleted_byte_count:15876189 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632036 + volume id:69 size:869213648 collection:"collection1" file_count:7293 delete_count:102 deleted_byte_count:30643207 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630534 + volume id:74 size:957046128 collection:"collection1" file_count:6982 delete_count:258 deleted_byte_count:73054259 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631460 + volume id:80 size:827912928 collection:"collection1" file_count:6914 delete_count:17 deleted_byte_count:5689635 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631157 + volume id:84 size:873121856 collection:"collection1" file_count:8200 delete_count:13 deleted_byte_count:3131676 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631161 + volume id:85 size:1023869320 collection:"collection1" file_count:7788 delete_count:234 deleted_byte_count:78037967 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631723 + volume id:97 size:1053112992 collection:"collection1" file_count:6789 delete_count:50 deleted_byte_count:38894001 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631193 + volume id:98 size:1077836440 collection:"collection1" file_count:7605 delete_count:202 deleted_byte_count:73180379 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615523691 + volume id:105 size:1073996824 collection:"collection1" file_count:6872 delete_count:20 deleted_byte_count:14482293 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615499757 + volume id:106 size:1075458664 collection:"collection1" file_count:7182 delete_count:307 deleted_byte_count:69349053 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615598137 + volume id:112 size:1076392512 collection:"collection1" file_count:8291 delete_count:156 deleted_byte_count:74120183 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615569823 + volume id:116 size:1074489504 collection:"collection1" file_count:9981 delete_count:174 deleted_byte_count:53998777 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615611565 + volume id:119 size:1075940104 collection:"collection1" file_count:9003 delete_count:12 deleted_byte_count:9128155 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615573878 + volume id:128 size:1074874632 collection:"collection1" file_count:9821 delete_count:148 deleted_byte_count:43633334 replica_placement:100 version:3 modified_at_second:1615602670 + volume id:133 size:1075952760 collection:"collection1" file_count:9538 delete_count:74 deleted_byte_count:19558008 replica_placement:100 version:3 modified_at_second:1615584779 + volume id:136 size:1074433552 collection:"collection1" file_count:9593 delete_count:72 deleted_byte_count:26912512 replica_placement:100 version:3 modified_at_second:1615376036 + volume id:138 size:1074465744 collection:"collection1" file_count:10120 delete_count:55 deleted_byte_count:15875438 replica_placement:100 version:3 modified_at_second:1615572231 + volume id:140 size:1076203744 collection:"collection1" file_count:11219 delete_count:57 deleted_byte_count:19864498 replica_placement:100 version:3 modified_at_second:1615571947 + volume id:144 size:1074549720 collection:"collection1" file_count:8780 delete_count:50 deleted_byte_count:52475146 replica_placement:100 version:3 modified_at_second:1615573451 + volume id:161 size:1077397192 collection:"collection1" file_count:9988 delete_count:28 deleted_byte_count:12509164 replica_placement:100 version:3 modified_at_second:1615631452 + volume id:173 size:1074154704 collection:"collection1" file_count:30884 delete_count:34 deleted_byte_count:2578509 replica_placement:100 version:3 modified_at_second:1615591904 + volume id:174 size:1073824232 collection:"collection1" file_count:30689 delete_count:36 deleted_byte_count:2160116 replica_placement:100 version:3 modified_at_second:1615598914 + volume id:197 size:1075423240 collection:"collection1" file_count:16473 delete_count:15 deleted_byte_count:12552442 replica_placement:100 version:3 modified_at_second:1615485254 + volume id:219 size:1092298904 collection:"collection1" file_count:3193 delete_count:17 deleted_byte_count:2047576 replica_placement:100 version:3 modified_at_second:1615579316 + volume id:263 size:1077167352 collection:"collection2" file_count:20227 delete_count:4 deleted_byte_count:97887 replica_placement:100 version:3 modified_at_second:1614871567 + volume id:272 size:1076146040 collection:"collection2" file_count:21034 delete_count:2 deleted_byte_count:216564 replica_placement:100 version:3 modified_at_second:1614884139 + volume id:291 size:1076256760 collection:"collection2" file_count:28301 delete_count:5 deleted_byte_count:116027 replica_placement:100 version:3 modified_at_second:1614904924 + volume id:299 size:1075147824 collection:"collection2" file_count:22927 delete_count:4 deleted_byte_count:345569 replica_placement:100 version:3 modified_at_second:1614918454 + volume id:301 size:1074655600 collection:"collection2" file_count:22543 delete_count:6 deleted_byte_count:136968 replica_placement:100 version:3 modified_at_second:1614918378 + volume id:302 size:1077559792 collection:"collection2" file_count:23124 delete_count:7 deleted_byte_count:293111 replica_placement:100 version:3 modified_at_second:1614925500 + volume id:339 size:1078402392 collection:"collection2" file_count:22309 replica_placement:100 version:3 modified_at_second:1614969996 + volume id:345 size:1074560760 collection:"collection2" file_count:22117 delete_count:2 deleted_byte_count:373286 replica_placement:100 version:3 modified_at_second:1614977458 + volume id:355 size:1075239792 collection:"collection2" file_count:22244 delete_count:1 deleted_byte_count:23282 replica_placement:100 version:3 modified_at_second:1614992157 + volume id:373 size:1080928000 collection:"collection2" file_count:22617 delete_count:4 deleted_byte_count:91849 replica_placement:100 version:3 modified_at_second:1615016877 + Disk hdd total size:48630015544 file_count:537880 deleted_file:2580 deleted_bytes:929560253 + DataNode 192.168.1.4:8080 total size:48630015544 file_count:537880 deleted_file:2580 deleted_bytes:929560253 + Rack rack1 total size:48630015544 file_count:537880 deleted_file:2580 deleted_bytes:929560253 + Rack rack2 hdd(volume:36/190 active:36 free:154 remote:0) + DataNode 192.168.1.2:8080 hdd(volume:36/190 active:36 free:154 remote:0) + Disk hdd(volume:36/190 active:36 free:154 remote:0) + volume id:2 size:289228560 file_count:1640 delete_count:4 deleted_byte_count:480564 replica_placement:100 version:3 compact_revision:6 modified_at_second:1615630622 + volume id:3 size:308743136 file_count:1638 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632242 + volume id:4 size:285986968 file_count:1641 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632302 + volume id:6 size:302411024 file_count:1604 delete_count:2 deleted_byte_count:274587 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631402 + volume id:7 size:1924728 collection:"collection4" file_count:15 replica_placement:100 version:3 modified_at_second:1609331040 + volume id:9 size:77337416 collection:"collection3" file_count:58 replica_placement:100 version:3 ttl:772 modified_at_second:1615513762 + volume id:10 size:1212784656 collection:"collection0" file_count:58 replica_placement:100 version:3 modified_at_second:1609814550 + volume id:12 size:1110923848 collection:"collection0" file_count:45 replica_placement:100 version:3 modified_at_second:1609819732 + volume id:13 size:1184910656 collection:"collection0" file_count:47 replica_placement:100 version:3 modified_at_second:1609827837 + volume id:14 size:1107475720 collection:"collection0" file_count:80 delete_count:3 deleted_byte_count:6870 replica_placement:100 version:3 modified_at_second:1612956980 + volume id:16 size:1113666104 collection:"collection0" file_count:73 delete_count:5 deleted_byte_count:6318 replica_placement:100 version:3 modified_at_second:1612957007 + volume id:17 size:1095115800 collection:"collection0" file_count:83 delete_count:3 deleted_byte_count:7099 replica_placement:100 version:3 modified_at_second:1612957000 + volume id:21 size:1097631664 collection:"collection0" file_count:82 delete_count:11 deleted_byte_count:68985100 replica_placement:100 version:3 modified_at_second:1612957007 + volume id:56 size:1001897616 collection:"collection1" file_count:8762 delete_count:37 deleted_byte_count:65375405 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632014 + volume id:81 size:880693104 collection:"collection1" file_count:7481 delete_count:236 deleted_byte_count:80386421 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631396 + volume id:104 size:1076383624 collection:"collection1" file_count:7663 delete_count:184 deleted_byte_count:100728071 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615602658 + volume id:107 size:1073811840 collection:"collection1" file_count:7436 delete_count:168 deleted_byte_count:57747484 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615293569 + volume id:113 size:1076709184 collection:"collection1" file_count:9355 delete_count:177 deleted_byte_count:59796765 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615569822 + volume id:139 size:1074163936 collection:"collection1" file_count:9315 delete_count:42 deleted_byte_count:10630966 replica_placement:100 version:3 modified_at_second:1615571946 + volume id:151 size:1098659752 collection:"collection1" file_count:10808 delete_count:24 deleted_byte_count:7088102 replica_placement:100 version:3 modified_at_second:1615586389 + volume id:155 size:1075140688 collection:"collection1" file_count:10882 delete_count:32 deleted_byte_count:9076141 replica_placement:100 version:3 modified_at_second:1615606614 + volume id:167 size:1073958176 collection:"collection1" file_count:25229 delete_count:48 deleted_byte_count:25871565 replica_placement:100 version:3 modified_at_second:1615602669 + volume id:177 size:1074120216 collection:"collection1" file_count:22293 delete_count:16 deleted_byte_count:3803952 replica_placement:100 version:3 modified_at_second:1615516892 + volume id:179 size:1074313920 collection:"collection1" file_count:21829 delete_count:24 deleted_byte_count:45552859 replica_placement:100 version:3 modified_at_second:1615580308 + volume id:182 size:1076131280 collection:"collection1" file_count:31987 delete_count:21 deleted_byte_count:1452346 replica_placement:100 version:3 modified_at_second:1615568922 + volume id:215 size:1068268216 collection:"collection1" file_count:2813 delete_count:10 deleted_byte_count:5676795 replica_placement:100 version:3 modified_at_second:1615586386 + volume id:217 size:1075381872 collection:"collection1" file_count:3331 delete_count:14 deleted_byte_count:2009141 replica_placement:100 version:3 modified_at_second:1615401638 + volume id:283 size:1080178944 collection:"collection2" file_count:19462 delete_count:7 deleted_byte_count:660407 replica_placement:100 version:3 modified_at_second:1614896626 + volume id:303 size:1075944504 collection:"collection2" file_count:22541 delete_count:2 deleted_byte_count:13617 replica_placement:100 version:3 modified_at_second:1614925431 + volume id:309 size:1075178624 collection:"collection2" file_count:22692 delete_count:3 deleted_byte_count:171124 replica_placement:100 version:3 modified_at_second:1614931409 + volume id:323 size:1074608200 collection:"collection2" file_count:21605 delete_count:4 deleted_byte_count:172090 replica_placement:100 version:3 modified_at_second:1614950526 + volume id:344 size:1075035448 collection:"collection2" file_count:21765 delete_count:1 deleted_byte_count:24623 replica_placement:100 version:3 modified_at_second:1614977465 + volume id:347 size:1075145496 collection:"collection2" file_count:22178 delete_count:1 deleted_byte_count:79392 replica_placement:100 version:3 modified_at_second:1614984727 + volume id:357 size:1074276208 collection:"collection2" file_count:23137 delete_count:4 deleted_byte_count:188487 replica_placement:100 version:3 modified_at_second:1614998792 + volume id:380 size:1010760456 collection:"collection2" file_count:14921 delete_count:6 deleted_byte_count:65678 replica_placement:100 version:3 modified_at_second:1615632322 + volume id:381 size:939292792 collection:"collection2" file_count:14619 delete_count:2 deleted_byte_count:5119 replica_placement:100 version:3 modified_at_second:1615632324 + Disk hdd total size:33468194376 file_count:369168 deleted_file:1091 deleted_bytes:546337088 + DataNode 192.168.1.2:8080 total size:33468194376 file_count:369168 deleted_file:1091 deleted_bytes:546337088 + Rack rack2 total size:33468194376 file_count:369168 deleted_file:1091 deleted_bytes:546337088 + DataCenter dc2 total size:82098209920 file_count:907048 deleted_file:3671 deleted_bytes:1475897341 + DataCenter dc3 hdd(volume:108/1850 active:108 free:1742 remote:0) + Rack rack3 hdd(volume:108/1850 active:108 free:1742 remote:0) + DataNode 192.168.1.6:8080 hdd(volume:108/1850 active:108 free:1742 remote:0) + Disk hdd(volume:108/1850 active:108 free:1742 remote:0) + volume id:1 size:284685936 file_count:1557 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632062 + volume id:32 size:281390512 file_count:1496 delete_count:6 deleted_byte_count:546403 replica_placement:100 version:3 modified_at_second:1615632362 + volume id:47 size:444599784 collection:"collection1" file_count:709 delete_count:19 deleted_byte_count:11913451 replica_placement:100 version:3 modified_at_second:1615632397 + volume id:49 size:1078775288 collection:"collection1" file_count:9636 delete_count:22 deleted_byte_count:5625976 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630446 + volume id:68 size:898630584 collection:"collection1" file_count:6934 delete_count:95 deleted_byte_count:27460707 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632284 + volume id:88 size:1073767976 collection:"collection1" file_count:14995 delete_count:206 deleted_byte_count:81222360 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615629897 + volume id:202 size:1077533160 collection:"collection1" file_count:2847 delete_count:67 deleted_byte_count:65172985 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615588497 + volume id:203 size:1027316272 collection:"collection1" file_count:3040 delete_count:11 deleted_byte_count:3993230 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631728 + volume id:205 size:1078485304 collection:"collection1" file_count:2869 delete_count:43 deleted_byte_count:18290259 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615579314 + volume id:206 size:1082045848 collection:"collection1" file_count:2979 delete_count:225 deleted_byte_count:88220074 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615564274 + volume id:209 size:1074083592 collection:"collection1" file_count:3238 delete_count:4 deleted_byte_count:1494244 replica_placement:100 version:3 modified_at_second:1615419954 + volume id:211 size:1080610712 collection:"collection1" file_count:3247 delete_count:7 deleted_byte_count:1891456 replica_placement:100 version:3 modified_at_second:1615269124 + volume id:212 size:1078293360 collection:"collection1" file_count:3106 delete_count:6 deleted_byte_count:2085755 replica_placement:100 version:3 modified_at_second:1615586387 + volume id:213 size:1093587976 collection:"collection1" file_count:3681 delete_count:12 deleted_byte_count:3138791 replica_placement:100 version:3 modified_at_second:1615586387 + volume id:214 size:1074486992 collection:"collection1" file_count:3217 delete_count:10 deleted_byte_count:6392871 replica_placement:100 version:3 modified_at_second:1615586383 + volume id:216 size:1080073496 collection:"collection1" file_count:3316 delete_count:4 deleted_byte_count:179819 replica_placement:100 version:3 modified_at_second:1615586387 + volume id:222 size:1106623104 collection:"collection1" file_count:3273 delete_count:11 deleted_byte_count:2114627 replica_placement:100 version:3 modified_at_second:1615586243 + volume id:223 size:1075233064 collection:"collection1" file_count:2966 delete_count:9 deleted_byte_count:744001 replica_placement:100 version:3 modified_at_second:1615586244 + volume id:227 size:1106699896 collection:"collection1" file_count:2827 delete_count:20 deleted_byte_count:5496790 replica_placement:100 version:3 modified_at_second:1615609989 + volume id:229 size:1109855312 collection:"collection1" file_count:2857 delete_count:22 deleted_byte_count:2839883 replica_placement:100 version:3 modified_at_second:1615609988 + volume id:230 size:1080722984 collection:"collection1" file_count:2898 delete_count:15 deleted_byte_count:3929261 replica_placement:100 version:3 modified_at_second:1615610537 + volume id:231 size:1112917696 collection:"collection1" file_count:3151 delete_count:20 deleted_byte_count:2989828 replica_placement:100 version:3 modified_at_second:1615611350 + volume id:233 size:1080526464 collection:"collection1" file_count:3136 delete_count:61 deleted_byte_count:17991717 replica_placement:100 version:3 modified_at_second:1615611352 + volume id:234 size:1073835280 collection:"collection1" file_count:2965 delete_count:41 deleted_byte_count:4960354 replica_placement:100 version:3 modified_at_second:1615611351 + volume id:235 size:1075586104 collection:"collection1" file_count:2767 delete_count:33 deleted_byte_count:3216540 replica_placement:100 version:3 modified_at_second:1615611354 + volume id:237 size:375722792 collection:"collection1" file_count:736 delete_count:16 deleted_byte_count:4464870 replica_placement:100 version:3 modified_at_second:1615631727 + volume id:239 size:426569024 collection:"collection1" file_count:693 delete_count:19 deleted_byte_count:13020783 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630838 + volume id:241 size:380217424 collection:"collection1" file_count:633 delete_count:6 deleted_byte_count:1715768 replica_placement:100 version:3 modified_at_second:1615632006 + volume id:244 size:1080295352 collection:"collection2" file_count:10812 delete_count:1 deleted_byte_count:795 replica_placement:100 version:3 modified_at_second:1614852171 + volume id:245 size:1074597056 collection:"collection2" file_count:10371 delete_count:3 deleted_byte_count:209701 replica_placement:100 version:3 modified_at_second:1614852093 + volume id:246 size:1075998648 collection:"collection2" file_count:10365 delete_count:1 deleted_byte_count:13112 replica_placement:100 version:3 modified_at_second:1614852105 + volume id:248 size:1084301184 collection:"collection2" file_count:11217 delete_count:4 deleted_byte_count:746488 replica_placement:100 version:3 modified_at_second:1614856285 + volume id:249 size:1074819136 collection:"collection2" file_count:10763 delete_count:2 deleted_byte_count:271699 replica_placement:100 version:3 modified_at_second:1614856230 + volume id:251 size:1075684488 collection:"collection2" file_count:10847 replica_placement:100 version:3 modified_at_second:1614856270 + volume id:252 size:1075065208 collection:"collection2" file_count:14622 delete_count:2 deleted_byte_count:5228 replica_placement:100 version:3 modified_at_second:1614861196 + volume id:253 size:1087328816 collection:"collection2" file_count:14920 delete_count:3 deleted_byte_count:522994 replica_placement:100 version:3 modified_at_second:1614861255 + volume id:255 size:1079581640 collection:"collection2" file_count:14877 delete_count:3 deleted_byte_count:101223 replica_placement:100 version:3 modified_at_second:1614861233 + volume id:256 size:1074283592 collection:"collection2" file_count:14157 delete_count:1 deleted_byte_count:18156 replica_placement:100 version:3 modified_at_second:1614861100 + volume id:258 size:1075527216 collection:"collection2" file_count:18421 delete_count:4 deleted_byte_count:267833 replica_placement:100 version:3 modified_at_second:1614866420 + volume id:259 size:1075507776 collection:"collection2" file_count:18079 delete_count:2 deleted_byte_count:71992 replica_placement:100 version:3 modified_at_second:1614866381 + volume id:264 size:1081624192 collection:"collection2" file_count:21151 replica_placement:100 version:3 modified_at_second:1614871629 + volume id:265 size:1076401104 collection:"collection2" file_count:19932 delete_count:2 deleted_byte_count:160823 replica_placement:100 version:3 modified_at_second:1615629130 + volume id:266 size:1075617464 collection:"collection2" file_count:20075 delete_count:1 deleted_byte_count:1039 replica_placement:100 version:3 modified_at_second:1614871526 + volume id:267 size:1075699544 collection:"collection2" file_count:21039 delete_count:3 deleted_byte_count:59956 replica_placement:100 version:3 modified_at_second:1614877294 + volume id:268 size:1074490592 collection:"collection2" file_count:21698 delete_count:1 deleted_byte_count:33968 replica_placement:100 version:3 modified_at_second:1614877434 + volume id:269 size:1077552872 collection:"collection2" file_count:21875 delete_count:4 deleted_byte_count:347272 replica_placement:100 version:3 modified_at_second:1614877481 + volume id:270 size:1076876568 collection:"collection2" file_count:22057 delete_count:1 deleted_byte_count:43916 replica_placement:100 version:3 modified_at_second:1614877469 + volume id:275 size:1078349024 collection:"collection2" file_count:20808 delete_count:1 deleted_byte_count:1118 replica_placement:100 version:3 modified_at_second:1614884147 + volume id:277 size:1074956288 collection:"collection2" file_count:19260 delete_count:2 deleted_byte_count:172356 replica_placement:100 version:3 modified_at_second:1614889988 + volume id:278 size:1078798640 collection:"collection2" file_count:20597 delete_count:5 deleted_byte_count:400060 replica_placement:100 version:3 modified_at_second:1614890292 + volume id:279 size:1077325040 collection:"collection2" file_count:19671 delete_count:6 deleted_byte_count:379116 replica_placement:100 version:3 modified_at_second:1614890229 + volume id:280 size:1077432216 collection:"collection2" file_count:20286 delete_count:1 deleted_byte_count:879 replica_placement:100 version:3 modified_at_second:1614890262 + volume id:281 size:1077581096 collection:"collection2" file_count:20206 delete_count:3 deleted_byte_count:143964 replica_placement:100 version:3 modified_at_second:1614890237 + volume id:284 size:1074533384 collection:"collection2" file_count:22196 delete_count:4 deleted_byte_count:154683 replica_placement:100 version:3 modified_at_second:1614897231 + volume id:285 size:1082128688 collection:"collection2" file_count:21804 delete_count:1 deleted_byte_count:1064 replica_placement:100 version:3 modified_at_second:1614897165 + volume id:289 size:1075284256 collection:"collection2" file_count:29342 delete_count:5 deleted_byte_count:100454 replica_placement:100 version:3 modified_at_second:1614904977 + volume id:290 size:1074723792 collection:"collection2" file_count:28340 delete_count:4 deleted_byte_count:199064 replica_placement:100 version:3 modified_at_second:1614904924 + volume id:291 size:1076256768 collection:"collection2" file_count:28301 delete_count:5 deleted_byte_count:116027 replica_placement:100 version:3 modified_at_second:1614904924 + volume id:293 size:1075409792 collection:"collection2" file_count:26063 delete_count:4 deleted_byte_count:183834 replica_placement:100 version:3 modified_at_second:1614912235 + volume id:294 size:1075444048 collection:"collection2" file_count:26076 delete_count:4 deleted_byte_count:194914 replica_placement:100 version:3 modified_at_second:1614912220 + volume id:296 size:1077824032 collection:"collection2" file_count:26741 delete_count:4 deleted_byte_count:199906 replica_placement:100 version:3 modified_at_second:1614912301 + volume id:297 size:1080229136 collection:"collection2" file_count:23409 delete_count:5 deleted_byte_count:46268 replica_placement:100 version:3 modified_at_second:1614918481 + volume id:298 size:1075410136 collection:"collection2" file_count:23222 delete_count:2 deleted_byte_count:46110 replica_placement:100 version:3 modified_at_second:1614918474 + volume id:299 size:1075147936 collection:"collection2" file_count:22927 delete_count:4 deleted_byte_count:345569 replica_placement:100 version:3 modified_at_second:1614918455 + volume id:300 size:1076212392 collection:"collection2" file_count:22892 delete_count:2 deleted_byte_count:61320 replica_placement:100 version:3 modified_at_second:1614918464 + volume id:301 size:1074655600 collection:"collection2" file_count:22543 delete_count:6 deleted_byte_count:136968 replica_placement:100 version:3 modified_at_second:1614918378 + volume id:303 size:1075944480 collection:"collection2" file_count:22541 delete_count:2 deleted_byte_count:13617 replica_placement:100 version:3 modified_at_second:1614925431 + volume id:306 size:1074764016 collection:"collection2" file_count:22939 replica_placement:100 version:3 modified_at_second:1614925462 + volume id:307 size:1076568000 collection:"collection2" file_count:23377 delete_count:2 deleted_byte_count:25453 replica_placement:100 version:3 modified_at_second:1614931448 + volume id:308 size:1074022392 collection:"collection2" file_count:23086 delete_count:2 deleted_byte_count:2127 replica_placement:100 version:3 modified_at_second:1614931401 + volume id:309 size:1075178664 collection:"collection2" file_count:22692 delete_count:3 deleted_byte_count:171124 replica_placement:100 version:3 modified_at_second:1614931409 + volume id:310 size:1074761528 collection:"collection2" file_count:21441 delete_count:3 deleted_byte_count:13934 replica_placement:100 version:3 modified_at_second:1614931077 + volume id:314 size:1074670840 collection:"collection2" file_count:20964 delete_count:4 deleted_byte_count:304291 replica_placement:100 version:3 modified_at_second:1614937441 + volume id:315 size:1084153544 collection:"collection2" file_count:23638 delete_count:2 deleted_byte_count:53956 replica_placement:100 version:3 modified_at_second:1614937885 + volume id:317 size:1076215096 collection:"collection2" file_count:23572 delete_count:2 deleted_byte_count:1441356 replica_placement:100 version:3 modified_at_second:1614943965 + volume id:318 size:1075965168 collection:"collection2" file_count:22459 delete_count:2 deleted_byte_count:37778 replica_placement:100 version:3 modified_at_second:1614943862 + volume id:319 size:1073952880 collection:"collection2" file_count:22286 delete_count:2 deleted_byte_count:43421 replica_placement:100 version:3 modified_at_second:1614943810 + volume id:320 size:1082437792 collection:"collection2" file_count:21544 delete_count:3 deleted_byte_count:16712 replica_placement:100 version:3 modified_at_second:1614943599 + volume id:321 size:1081477904 collection:"collection2" file_count:23531 delete_count:5 deleted_byte_count:262564 replica_placement:100 version:3 modified_at_second:1614943982 + volume id:324 size:1075606680 collection:"collection2" file_count:20799 delete_count:1 deleted_byte_count:251210 replica_placement:100 version:3 modified_at_second:1614950310 + volume id:325 size:1080701144 collection:"collection2" file_count:21735 replica_placement:100 version:3 modified_at_second:1614950525 + volume id:330 size:1080825832 collection:"collection2" file_count:22464 delete_count:2 deleted_byte_count:15771 replica_placement:100 version:3 modified_at_second:1614956477 + volume id:332 size:1075569928 collection:"collection2" file_count:22097 delete_count:3 deleted_byte_count:98273 replica_placement:100 version:3 modified_at_second:1614962869 + volume id:334 size:1075607880 collection:"collection2" file_count:22546 delete_count:6 deleted_byte_count:101538 replica_placement:100 version:3 modified_at_second:1614962978 + volume id:336 size:1087853056 collection:"collection2" file_count:22801 delete_count:2 deleted_byte_count:26394 replica_placement:100 version:3 modified_at_second:1614963005 + volume id:337 size:1075646784 collection:"collection2" file_count:21934 delete_count:1 deleted_byte_count:3397 replica_placement:100 version:3 modified_at_second:1614969937 + volume id:338 size:1076118304 collection:"collection2" file_count:21680 replica_placement:100 version:3 modified_at_second:1614969850 + volume id:340 size:1079462184 collection:"collection2" file_count:22319 delete_count:4 deleted_byte_count:93620 replica_placement:100 version:3 modified_at_second:1614969978 + volume id:341 size:1074448400 collection:"collection2" file_count:21590 delete_count:5 deleted_byte_count:160085 replica_placement:100 version:3 modified_at_second:1614969858 + volume id:342 size:1080186424 collection:"collection2" file_count:22405 delete_count:4 deleted_byte_count:64819 replica_placement:100 version:3 modified_at_second:1614977521 + volume id:344 size:1075035416 collection:"collection2" file_count:21765 delete_count:1 deleted_byte_count:24623 replica_placement:100 version:3 modified_at_second:1614977465 + volume id:345 size:1074560760 collection:"collection2" file_count:22117 delete_count:2 deleted_byte_count:373286 replica_placement:100 version:3 modified_at_second:1614977457 + volume id:346 size:1076464112 collection:"collection2" file_count:22320 delete_count:4 deleted_byte_count:798258 replica_placement:100 version:3 modified_at_second:1615631322 + volume id:348 size:1080623640 collection:"collection2" file_count:21667 delete_count:1 deleted_byte_count:2443 replica_placement:100 version:3 modified_at_second:1614984606 + volume id:350 size:1074756688 collection:"collection2" file_count:21990 delete_count:3 deleted_byte_count:233881 replica_placement:100 version:3 modified_at_second:1614984682 + volume id:351 size:1078795112 collection:"collection2" file_count:23660 delete_count:3 deleted_byte_count:102141 replica_placement:100 version:3 modified_at_second:1614984816 + volume id:352 size:1077145936 collection:"collection2" file_count:22066 delete_count:1 deleted_byte_count:1018 replica_placement:100 version:3 modified_at_second:1614992130 + volume id:353 size:1074897496 collection:"collection2" file_count:21266 delete_count:2 deleted_byte_count:3105374 replica_placement:100 version:3 modified_at_second:1614991951 + volume id:355 size:1075239728 collection:"collection2" file_count:22244 delete_count:1 deleted_byte_count:23282 replica_placement:100 version:3 modified_at_second:1614992157 + volume id:356 size:1083305048 collection:"collection2" file_count:21552 delete_count:4 deleted_byte_count:14472 replica_placement:100 version:3 modified_at_second:1614992028 + volume id:358 size:1085152368 collection:"collection2" file_count:23756 delete_count:3 deleted_byte_count:44531 replica_placement:100 version:3 modified_at_second:1614998824 + volume id:360 size:1075532456 collection:"collection2" file_count:22574 delete_count:3 deleted_byte_count:1774776 replica_placement:100 version:3 modified_at_second:1614998770 + volume id:361 size:1075362744 collection:"collection2" file_count:22272 delete_count:1 deleted_byte_count:3497 replica_placement:100 version:3 modified_at_second:1614998668 + volume id:375 size:1076140568 collection:"collection2" file_count:21880 delete_count:2 deleted_byte_count:51103 replica_placement:100 version:3 modified_at_second:1615016787 + volume id:376 size:1074845944 collection:"collection2" file_count:22908 delete_count:4 deleted_byte_count:432305 replica_placement:100 version:3 modified_at_second:1615016916 + volume id:377 size:957284144 collection:"collection2" file_count:14923 delete_count:1 deleted_byte_count:1797 replica_placement:100 version:3 modified_at_second:1615632323 + volume id:378 size:959273936 collection:"collection2" file_count:15027 delete_count:4 deleted_byte_count:231414 replica_placement:100 version:3 modified_at_second:1615632323 + volume id:381 size:939261032 collection:"collection2" file_count:14615 delete_count:5 deleted_byte_count:1192272 replica_placement:100 version:3 modified_at_second:1615632324 + Disk hdd total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585 + DataNode 192.168.1.6:8080 total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585 + Rack rack3 total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585 + DataCenter dc3 total size:111617646696 file_count:1762773 deleted_file:1221 deleted_bytes:398484585 + DataCenter dc4 hdd(volume:267/2000 active:267 free:1733 remote:0) + Rack DefaultRack hdd(volume:267/2000 active:267 free:1733 remote:0) + DataNode 192.168.1.1:8080 hdd(volume:267/2000 active:267 free:1733 remote:0) + Disk hdd(volume:267/2000 active:267 free:1733 remote:0) + volume id:1 size:284693256 file_count:1558 delete_count:2 deleted_byte_count:4818 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632062 + volume id:2 size:289228560 file_count:1640 delete_count:4 deleted_byte_count:464508 replica_placement:100 version:3 compact_revision:6 modified_at_second:1615630622 + volume id:3 size:308741952 file_count:1637 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632242 + volume id:4 size:285986968 file_count:1640 delete_count:1 deleted_byte_count:145095 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632302 + volume id:5 size:293806008 file_count:1669 delete_count:2 deleted_byte_count:274334 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631342 + volume id:6 size:302411024 file_count:1604 delete_count:2 deleted_byte_count:274587 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631402 + volume id:7 size:1924728 collection:"collection4" file_count:15 replica_placement:100 version:3 modified_at_second:1609331040 + volume id:9 size:77337416 collection:"collection3" file_count:58 replica_placement:100 version:3 ttl:772 modified_at_second:1615513762 + volume id:10 size:1212784656 collection:"collection0" file_count:58 replica_placement:100 version:3 modified_at_second:1609814543 + volume id:11 size:1109224552 collection:"collection0" file_count:44 replica_placement:100 version:3 modified_at_second:1609815123 + volume id:12 size:1110923848 collection:"collection0" file_count:45 replica_placement:100 version:3 modified_at_second:1609819726 + volume id:13 size:1184910656 collection:"collection0" file_count:47 replica_placement:100 version:3 modified_at_second:1609827832 + volume id:14 size:1107475720 collection:"collection0" file_count:80 delete_count:3 deleted_byte_count:6870 replica_placement:100 version:3 modified_at_second:1612956983 + volume id:15 size:1115965160 collection:"collection0" file_count:83 delete_count:3 deleted_byte_count:4956 replica_placement:100 version:3 modified_at_second:1612957001 + volume id:16 size:1113666048 collection:"collection0" file_count:73 delete_count:5 deleted_byte_count:6318 replica_placement:100 version:3 modified_at_second:1612957007 + volume id:17 size:1095115800 collection:"collection0" file_count:83 delete_count:3 deleted_byte_count:7099 replica_placement:100 version:3 modified_at_second:1612957000 + volume id:18 size:1096678688 collection:"collection0" file_count:88 delete_count:4 deleted_byte_count:8633 replica_placement:100 version:3 modified_at_second:1612957000 + volume id:19 size:1096923792 collection:"collection0" file_count:100 delete_count:10 deleted_byte_count:75779917 replica_placement:100 version:3 compact_revision:4 modified_at_second:1612957011 + volume id:20 size:1074760432 collection:"collection0" file_count:82 delete_count:5 deleted_byte_count:12156 replica_placement:100 version:3 compact_revision:2 modified_at_second:1612957011 + volume id:22 size:1086828368 collection:"collection0" file_count:75 delete_count:3 deleted_byte_count:5551 replica_placement:100 version:3 modified_at_second:1612957007 + volume id:23 size:1076380280 collection:"collection0" file_count:68 delete_count:2 deleted_byte_count:2910 replica_placement:100 version:3 modified_at_second:1612957011 + volume id:24 size:1074139808 collection:"collection0" file_count:90 delete_count:1 deleted_byte_count:1977 replica_placement:100 version:3 modified_at_second:1612957011 + volume id:25 size:690757544 collection:"collection0" file_count:38 delete_count:1 deleted_byte_count:1944 replica_placement:100 version:3 modified_at_second:1612956995 + volume id:26 size:532657632 collection:"collection0" file_count:100 delete_count:4 deleted_byte_count:9081 replica_placement:100 version:3 modified_at_second:1614170023 + volume id:34 size:1077111136 collection:"collection1" file_count:9781 delete_count:110 deleted_byte_count:20894827 replica_placement:100 version:3 modified_at_second:1615619366 + volume id:35 size:1075241656 collection:"collection1" file_count:10523 delete_count:96 deleted_byte_count:46618989 replica_placement:100 version:3 modified_at_second:1615618790 + volume id:36 size:1075118360 collection:"collection1" file_count:10342 delete_count:116 deleted_byte_count:25493106 replica_placement:100 version:3 modified_at_second:1615606148 + volume id:37 size:1075895584 collection:"collection1" file_count:12013 delete_count:98 deleted_byte_count:50747932 replica_placement:100 version:3 modified_at_second:1615594777 + volume id:39 size:1076606536 collection:"collection1" file_count:12612 delete_count:78 deleted_byte_count:17462730 replica_placement:100 version:3 modified_at_second:1615611959 + volume id:40 size:1075358552 collection:"collection1" file_count:12597 delete_count:62 deleted_byte_count:11657901 replica_placement:100 version:3 modified_at_second:1615612994 + volume id:41 size:1076283528 collection:"collection1" file_count:12088 delete_count:84 deleted_byte_count:19319268 replica_placement:100 version:3 modified_at_second:1615596736 + volume id:42 size:1093948352 collection:"collection1" file_count:7889 delete_count:47 deleted_byte_count:5697275 replica_placement:100 version:3 modified_at_second:1615548908 + volume id:43 size:1116445864 collection:"collection1" file_count:7358 delete_count:54 deleted_byte_count:9534379 replica_placement:100 version:3 modified_at_second:1615566170 + volume id:44 size:1077582560 collection:"collection1" file_count:7295 delete_count:50 deleted_byte_count:12618414 replica_placement:100 version:3 modified_at_second:1615566170 + volume id:45 size:1075254640 collection:"collection1" file_count:10772 delete_count:76 deleted_byte_count:22426345 replica_placement:100 version:3 modified_at_second:1615573499 + volume id:46 size:1075286056 collection:"collection1" file_count:9947 delete_count:309 deleted_byte_count:105601163 replica_placement:100 version:3 modified_at_second:1615569826 + volume id:48 size:1076778720 collection:"collection1" file_count:9850 delete_count:77 deleted_byte_count:16641907 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630690 + volume id:50 size:1076688224 collection:"collection1" file_count:7921 delete_count:26 deleted_byte_count:5162032 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615610879 + volume id:52 size:1083529704 collection:"collection1" file_count:10128 delete_count:32 deleted_byte_count:10608391 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615599195 + volume id:53 size:1063089216 collection:"collection1" file_count:9832 delete_count:31 deleted_byte_count:9273066 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632156 + volume id:55 size:1012890016 collection:"collection1" file_count:8651 delete_count:27 deleted_byte_count:9418841 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631452 + volume id:57 size:839849792 collection:"collection1" file_count:7514 delete_count:24 deleted_byte_count:6228543 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631774 + volume id:58 size:908064200 collection:"collection1" file_count:8128 delete_count:21 deleted_byte_count:6113731 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632342 + volume id:59 size:988302272 collection:"collection1" file_count:8098 delete_count:20 deleted_byte_count:3947615 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632238 + volume id:60 size:1010702480 collection:"collection1" file_count:8969 delete_count:79 deleted_byte_count:24782814 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632439 + volume id:61 size:975604488 collection:"collection1" file_count:8683 delete_count:20 deleted_byte_count:10276072 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615631176 + volume id:62 size:873845936 collection:"collection1" file_count:7897 delete_count:23 deleted_byte_count:10920170 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631133 + volume id:64 size:965638488 collection:"collection1" file_count:8218 delete_count:27 deleted_byte_count:6922489 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631031 + volume id:65 size:823283552 collection:"collection1" file_count:7834 delete_count:29 deleted_byte_count:5950610 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632306 + volume id:66 size:821343440 collection:"collection1" file_count:7383 delete_count:29 deleted_byte_count:12010343 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631968 + volume id:67 size:878713872 collection:"collection1" file_count:7299 delete_count:117 deleted_byte_count:24857326 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632156 + volume id:68 size:898630584 collection:"collection1" file_count:6934 delete_count:95 deleted_byte_count:27460707 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632284 + volume id:70 size:886695472 collection:"collection1" file_count:7769 delete_count:164 deleted_byte_count:45162513 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632398 + volume id:71 size:907608392 collection:"collection1" file_count:7658 delete_count:122 deleted_byte_count:27622941 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632307 + volume id:72 size:903990720 collection:"collection1" file_count:6996 delete_count:240 deleted_byte_count:74147727 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630982 + volume id:73 size:929047664 collection:"collection1" file_count:7038 delete_count:227 deleted_byte_count:65336664 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630707 + volume id:74 size:957046128 collection:"collection1" file_count:6981 delete_count:259 deleted_byte_count:73080838 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631460 + volume id:75 size:908044992 collection:"collection1" file_count:6911 delete_count:268 deleted_byte_count:73934373 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632430 + volume id:76 size:985296744 collection:"collection1" file_count:6566 delete_count:61 deleted_byte_count:44464430 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632284 + volume id:77 size:929398296 collection:"collection1" file_count:7427 delete_count:238 deleted_byte_count:59581579 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632013 + volume id:78 size:1075671512 collection:"collection1" file_count:7540 delete_count:258 deleted_byte_count:71726846 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615582829 + volume id:79 size:948225472 collection:"collection1" file_count:6997 delete_count:227 deleted_byte_count:60625763 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631326 + volume id:82 size:1041661800 collection:"collection1" file_count:7043 delete_count:207 deleted_byte_count:52275724 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632430 + volume id:83 size:936195856 collection:"collection1" file_count:7593 delete_count:13 deleted_byte_count:4633917 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632029 + volume id:85 size:1023867520 collection:"collection1" file_count:7787 delete_count:240 deleted_byte_count:82091742 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631723 + volume id:86 size:1009437488 collection:"collection1" file_count:8474 delete_count:236 deleted_byte_count:64543674 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630812 + volume id:87 size:922276640 collection:"collection1" file_count:12902 delete_count:13 deleted_byte_count:3412959 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632438 + volume id:89 size:1044401976 collection:"collection1" file_count:14943 delete_count:243 deleted_byte_count:58543159 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632208 + volume id:90 size:891145784 collection:"collection1" file_count:14608 delete_count:10 deleted_byte_count:2564369 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615629390 + volume id:91 size:936572832 collection:"collection1" file_count:14686 delete_count:11 deleted_byte_count:4717727 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631851 + volume id:92 size:992440712 collection:"collection1" file_count:7061 delete_count:195 deleted_byte_count:60649573 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630566 + volume id:93 size:1079603768 collection:"collection1" file_count:7878 delete_count:270 deleted_byte_count:74150048 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615556015 + volume id:94 size:1030685824 collection:"collection1" file_count:7660 delete_count:207 deleted_byte_count:70150733 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631616 + volume id:95 size:990879168 collection:"collection1" file_count:6620 delete_count:206 deleted_byte_count:60363604 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615631866 + volume id:96 size:989296136 collection:"collection1" file_count:7544 delete_count:229 deleted_byte_count:59931853 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630778 + volume id:97 size:1053112992 collection:"collection1" file_count:6789 delete_count:50 deleted_byte_count:38894001 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631194 + volume id:99 size:1071718504 collection:"collection1" file_count:7470 delete_count:8 deleted_byte_count:9624950 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631175 + volume id:100 size:1083617440 collection:"collection1" file_count:7018 delete_count:187 deleted_byte_count:61304236 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615505917 + volume id:101 size:1077109520 collection:"collection1" file_count:7706 delete_count:226 deleted_byte_count:77864841 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630994 + volume id:102 size:1074359920 collection:"collection1" file_count:7338 delete_count:7 deleted_byte_count:6499151 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615626683 + volume id:103 size:1075863904 collection:"collection1" file_count:7184 delete_count:186 deleted_byte_count:58872238 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615628417 + volume id:104 size:1076383768 collection:"collection1" file_count:7663 delete_count:184 deleted_byte_count:100578087 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615602661 + volume id:105 size:1073996824 collection:"collection1" file_count:6873 delete_count:19 deleted_byte_count:14271533 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615499756 + volume id:108 size:1074648024 collection:"collection1" file_count:7472 delete_count:194 deleted_byte_count:70864699 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615593232 + volume id:109 size:1075254560 collection:"collection1" file_count:7556 delete_count:263 deleted_byte_count:55155265 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615502487 + volume id:110 size:1076575744 collection:"collection1" file_count:6996 delete_count:163 deleted_byte_count:52954032 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615590786 + volume id:111 size:1073826232 collection:"collection1" file_count:7355 delete_count:155 deleted_byte_count:50083578 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615593233 + volume id:114 size:1074762784 collection:"collection1" file_count:8802 delete_count:156 deleted_byte_count:38470055 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615591826 + volume id:115 size:1076192240 collection:"collection1" file_count:7690 delete_count:154 deleted_byte_count:32267193 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615285295 + volume id:116 size:1074489504 collection:"collection1" file_count:9981 delete_count:174 deleted_byte_count:53998777 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615611567 + volume id:117 size:1073917192 collection:"collection1" file_count:9520 delete_count:114 deleted_byte_count:21835126 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615573714 + volume id:118 size:1074064400 collection:"collection1" file_count:8738 delete_count:15 deleted_byte_count:3460697 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615516265 + volume id:119 size:1075940104 collection:"collection1" file_count:9003 delete_count:12 deleted_byte_count:9128155 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615573880 + volume id:120 size:1076115928 collection:"collection1" file_count:9639 delete_count:118 deleted_byte_count:33357871 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615482567 + volume id:121 size:1078803248 collection:"collection1" file_count:10113 delete_count:441 deleted_byte_count:94128627 replica_placement:100 version:3 modified_at_second:1615506629 + volume id:122 size:1076235312 collection:"collection1" file_count:9106 delete_count:252 deleted_byte_count:93041272 replica_placement:100 version:3 modified_at_second:1615585913 + volume id:123 size:1080491112 collection:"collection1" file_count:10623 delete_count:302 deleted_byte_count:83956998 replica_placement:100 version:3 modified_at_second:1615585916 + volume id:124 size:1074519360 collection:"collection1" file_count:9457 delete_count:286 deleted_byte_count:74752459 replica_placement:100 version:3 modified_at_second:1615585916 + volume id:125 size:1088687040 collection:"collection1" file_count:9518 delete_count:281 deleted_byte_count:76037905 replica_placement:100 version:3 modified_at_second:1615585913 + volume id:126 size:1073867464 collection:"collection1" file_count:9320 delete_count:278 deleted_byte_count:94547424 replica_placement:100 version:3 modified_at_second:1615585912 + volume id:127 size:1074907336 collection:"collection1" file_count:9900 delete_count:133 deleted_byte_count:48570820 replica_placement:100 version:3 modified_at_second:1615612991 + volume id:129 size:1074704272 collection:"collection1" file_count:10012 delete_count:150 deleted_byte_count:64491721 replica_placement:100 version:3 modified_at_second:1615627566 + volume id:130 size:1075000632 collection:"collection1" file_count:10633 delete_count:161 deleted_byte_count:34768201 replica_placement:100 version:3 modified_at_second:1615582330 + volume id:131 size:1075279584 collection:"collection1" file_count:10075 delete_count:135 deleted_byte_count:29795712 replica_placement:100 version:3 modified_at_second:1615523898 + volume id:132 size:1088539496 collection:"collection1" file_count:11051 delete_count:71 deleted_byte_count:17178322 replica_placement:100 version:3 modified_at_second:1615619584 + volume id:133 size:1075952760 collection:"collection1" file_count:9538 delete_count:74 deleted_byte_count:19558008 replica_placement:100 version:3 modified_at_second:1615584780 + volume id:134 size:1074367304 collection:"collection1" file_count:10662 delete_count:69 deleted_byte_count:25530139 replica_placement:100 version:3 modified_at_second:1615555876 + volume id:135 size:1073906720 collection:"collection1" file_count:10446 delete_count:71 deleted_byte_count:28599975 replica_placement:100 version:3 modified_at_second:1615569816 + volume id:137 size:1074309264 collection:"collection1" file_count:9633 delete_count:50 deleted_byte_count:27487972 replica_placement:100 version:3 modified_at_second:1615572231 + volume id:139 size:1074163936 collection:"collection1" file_count:9314 delete_count:43 deleted_byte_count:10631353 replica_placement:100 version:3 modified_at_second:1615571946 + volume id:141 size:1074619488 collection:"collection1" file_count:9840 delete_count:45 deleted_byte_count:40890181 replica_placement:100 version:3 modified_at_second:1615630994 + volume id:142 size:1075732992 collection:"collection1" file_count:9009 delete_count:48 deleted_byte_count:9912854 replica_placement:100 version:3 modified_at_second:1615598914 + volume id:143 size:1075011280 collection:"collection1" file_count:9608 delete_count:51 deleted_byte_count:37282460 replica_placement:100 version:3 modified_at_second:1615488586 + volume id:145 size:1074394928 collection:"collection1" file_count:9255 delete_count:34 deleted_byte_count:38011392 replica_placement:100 version:3 modified_at_second:1615591825 + volume id:146 size:1076337520 collection:"collection1" file_count:10492 delete_count:50 deleted_byte_count:17071505 replica_placement:100 version:3 modified_at_second:1615632005 + volume id:147 size:1077130544 collection:"collection1" file_count:10451 delete_count:27 deleted_byte_count:8290907 replica_placement:100 version:3 modified_at_second:1615604117 + volume id:148 size:1076066568 collection:"collection1" file_count:9547 delete_count:33 deleted_byte_count:7034089 replica_placement:100 version:3 modified_at_second:1615586393 + volume id:149 size:1074989016 collection:"collection1" file_count:8352 delete_count:35 deleted_byte_count:7179742 replica_placement:100 version:3 modified_at_second:1615494496 + volume id:150 size:1076290408 collection:"collection1" file_count:9328 delete_count:33 deleted_byte_count:43417791 replica_placement:100 version:3 modified_at_second:1615611569 + volume id:151 size:1098659752 collection:"collection1" file_count:10805 delete_count:27 deleted_byte_count:7209106 replica_placement:100 version:3 modified_at_second:1615586390 + volume id:152 size:1075941376 collection:"collection1" file_count:9951 delete_count:36 deleted_byte_count:25348335 replica_placement:100 version:3 modified_at_second:1615606614 + volume id:153 size:1078539784 collection:"collection1" file_count:10924 delete_count:34 deleted_byte_count:12603081 replica_placement:100 version:3 modified_at_second:1615606614 + volume id:154 size:1081244752 collection:"collection1" file_count:11002 delete_count:31 deleted_byte_count:8467560 replica_placement:100 version:3 modified_at_second:1615478471 + volume id:156 size:1074975832 collection:"collection1" file_count:9535 delete_count:40 deleted_byte_count:11426621 replica_placement:100 version:3 modified_at_second:1615628342 + volume id:157 size:1076758536 collection:"collection1" file_count:10012 delete_count:19 deleted_byte_count:11688737 replica_placement:100 version:3 modified_at_second:1615597782 + volume id:158 size:1087251976 collection:"collection1" file_count:9972 delete_count:20 deleted_byte_count:10328429 replica_placement:100 version:3 modified_at_second:1615588879 + volume id:159 size:1074132336 collection:"collection1" file_count:9382 delete_count:27 deleted_byte_count:11474574 replica_placement:100 version:3 modified_at_second:1615593593 + volume id:160 size:1075680976 collection:"collection1" file_count:9772 delete_count:22 deleted_byte_count:4981968 replica_placement:100 version:3 modified_at_second:1615597782 + volume id:161 size:1077397136 collection:"collection1" file_count:9988 delete_count:28 deleted_byte_count:12509164 replica_placement:100 version:3 modified_at_second:1615631452 + volume id:162 size:1074286880 collection:"collection1" file_count:11220 delete_count:17 deleted_byte_count:1815547 replica_placement:100 version:3 modified_at_second:1615478127 + volume id:163 size:1074457224 collection:"collection1" file_count:12524 delete_count:27 deleted_byte_count:6359619 replica_placement:100 version:3 modified_at_second:1615579313 + volume id:164 size:1074261256 collection:"collection1" file_count:11922 delete_count:25 deleted_byte_count:2923288 replica_placement:100 version:3 modified_at_second:1615620085 + volume id:165 size:1073891080 collection:"collection1" file_count:9152 delete_count:12 deleted_byte_count:19164659 replica_placement:100 version:3 modified_at_second:1615471910 + volume id:166 size:1075637536 collection:"collection1" file_count:14211 delete_count:24 deleted_byte_count:20415490 replica_placement:100 version:3 modified_at_second:1615491021 + volume id:167 size:1073958280 collection:"collection1" file_count:25231 delete_count:48 deleted_byte_count:26022344 replica_placement:100 version:3 modified_at_second:1615620014 + volume id:168 size:1074718864 collection:"collection1" file_count:25702 delete_count:40 deleted_byte_count:4024775 replica_placement:100 version:3 modified_at_second:1615585664 + volume id:169 size:1073863032 collection:"collection1" file_count:25248 delete_count:43 deleted_byte_count:3013817 replica_placement:100 version:3 modified_at_second:1615569832 + volume id:170 size:1075747088 collection:"collection1" file_count:24596 delete_count:41 deleted_byte_count:3494711 replica_placement:100 version:3 modified_at_second:1615579207 + volume id:171 size:1081881400 collection:"collection1" file_count:24215 delete_count:36 deleted_byte_count:3191335 replica_placement:100 version:3 modified_at_second:1615596486 + volume id:172 size:1074787304 collection:"collection1" file_count:31236 delete_count:50 deleted_byte_count:3316482 replica_placement:100 version:3 modified_at_second:1615612385 + volume id:174 size:1073824160 collection:"collection1" file_count:30689 delete_count:36 deleted_byte_count:2160116 replica_placement:100 version:3 modified_at_second:1615598914 + volume id:175 size:1077742472 collection:"collection1" file_count:32353 delete_count:33 deleted_byte_count:1861403 replica_placement:100 version:3 modified_at_second:1615559516 + volume id:176 size:1073854800 collection:"collection1" file_count:30582 delete_count:34 deleted_byte_count:7701976 replica_placement:100 version:3 modified_at_second:1615626169 + volume id:178 size:1087560112 collection:"collection1" file_count:23482 delete_count:22 deleted_byte_count:18810492 replica_placement:100 version:3 modified_at_second:1615541369 + volume id:179 size:1074313920 collection:"collection1" file_count:21829 delete_count:24 deleted_byte_count:45574435 replica_placement:100 version:3 modified_at_second:1615580308 + volume id:180 size:1078438448 collection:"collection1" file_count:23614 delete_count:12 deleted_byte_count:4496474 replica_placement:100 version:3 modified_at_second:1614773243 + volume id:181 size:1074571672 collection:"collection1" file_count:22898 delete_count:19 deleted_byte_count:6628413 replica_placement:100 version:3 modified_at_second:1614745117 + volume id:183 size:1076361616 collection:"collection1" file_count:31293 delete_count:16 deleted_byte_count:468841 replica_placement:100 version:3 modified_at_second:1615572985 + volume id:184 size:1074594216 collection:"collection1" file_count:31368 delete_count:22 deleted_byte_count:857453 replica_placement:100 version:3 modified_at_second:1615586578 + volume id:185 size:1074099592 collection:"collection1" file_count:30612 delete_count:17 deleted_byte_count:2610847 replica_placement:100 version:3 modified_at_second:1615506835 + volume id:186 size:1074220664 collection:"collection1" file_count:31450 delete_count:15 deleted_byte_count:391855 replica_placement:100 version:3 modified_at_second:1615614934 + volume id:187 size:1074396112 collection:"collection1" file_count:31853 delete_count:17 deleted_byte_count:454283 replica_placement:100 version:3 modified_at_second:1615590491 + volume id:188 size:1074732632 collection:"collection1" file_count:31867 delete_count:19 deleted_byte_count:393743 replica_placement:100 version:3 modified_at_second:1615487645 + volume id:189 size:1074847824 collection:"collection1" file_count:31450 delete_count:16 deleted_byte_count:1040552 replica_placement:100 version:3 modified_at_second:1615335661 + volume id:190 size:1074008968 collection:"collection1" file_count:31987 delete_count:11 deleted_byte_count:685125 replica_placement:100 version:3 modified_at_second:1615447162 + volume id:191 size:1075492960 collection:"collection1" file_count:31301 delete_count:19 deleted_byte_count:708401 replica_placement:100 version:3 modified_at_second:1615357457 + volume id:192 size:1075857384 collection:"collection1" file_count:31490 delete_count:25 deleted_byte_count:720617 replica_placement:100 version:3 modified_at_second:1615621632 + volume id:193 size:1076616760 collection:"collection1" file_count:31907 delete_count:16 deleted_byte_count:464900 replica_placement:100 version:3 modified_at_second:1615507877 + volume id:194 size:1073985792 collection:"collection1" file_count:31434 delete_count:18 deleted_byte_count:391432 replica_placement:100 version:3 modified_at_second:1615559502 + volume id:195 size:1074158304 collection:"collection1" file_count:31453 delete_count:15 deleted_byte_count:718266 replica_placement:100 version:3 modified_at_second:1615559331 + volume id:196 size:1074594640 collection:"collection1" file_count:31665 delete_count:18 deleted_byte_count:3468922 replica_placement:100 version:3 modified_at_second:1615501690 + volume id:198 size:1075104624 collection:"collection1" file_count:16577 delete_count:18 deleted_byte_count:6583181 replica_placement:100 version:3 modified_at_second:1615623371 + volume id:199 size:1078117688 collection:"collection1" file_count:16497 delete_count:14 deleted_byte_count:1514286 replica_placement:100 version:3 modified_at_second:1615585987 + volume id:200 size:1075630464 collection:"collection1" file_count:16380 delete_count:18 deleted_byte_count:1103109 replica_placement:100 version:3 modified_at_second:1615485252 + volume id:201 size:1091460440 collection:"collection1" file_count:16684 delete_count:26 deleted_byte_count:5590335 replica_placement:100 version:3 modified_at_second:1615585987 + volume id:204 size:1079766904 collection:"collection1" file_count:3233 delete_count:255 deleted_byte_count:104707641 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615565702 + volume id:207 size:1081939960 collection:"collection1" file_count:3010 delete_count:4 deleted_byte_count:692350 replica_placement:100 version:3 modified_at_second:1615269061 + volume id:208 size:1077863624 collection:"collection1" file_count:3147 delete_count:6 deleted_byte_count:858726 replica_placement:100 version:3 modified_at_second:1615495515 + volume id:209 size:1074083592 collection:"collection1" file_count:3238 delete_count:4 deleted_byte_count:1494244 replica_placement:100 version:3 modified_at_second:1615419954 + volume id:210 size:1094311304 collection:"collection1" file_count:3468 delete_count:4 deleted_byte_count:466433 replica_placement:100 version:3 modified_at_second:1615495515 + volume id:211 size:1080610712 collection:"collection1" file_count:3247 delete_count:7 deleted_byte_count:1891456 replica_placement:100 version:3 modified_at_second:1615269124 + volume id:216 size:1080073496 collection:"collection1" file_count:3316 delete_count:4 deleted_byte_count:179819 replica_placement:100 version:3 modified_at_second:1615586387 + volume id:218 size:1081263944 collection:"collection1" file_count:3433 delete_count:14 deleted_byte_count:3454237 replica_placement:100 version:3 modified_at_second:1615603637 + volume id:220 size:1081928312 collection:"collection1" file_count:3166 delete_count:13 deleted_byte_count:4127709 replica_placement:100 version:3 modified_at_second:1615579317 + volume id:221 size:1106545536 collection:"collection1" file_count:3153 delete_count:11 deleted_byte_count:1496835 replica_placement:100 version:3 modified_at_second:1615269138 + volume id:224 size:1093691520 collection:"collection1" file_count:3463 delete_count:10 deleted_byte_count:1128328 replica_placement:100 version:3 modified_at_second:1615601870 + volume id:225 size:1080698928 collection:"collection1" file_count:3115 delete_count:7 deleted_byte_count:18170416 replica_placement:100 version:3 modified_at_second:1615434685 + volume id:226 size:1103504792 collection:"collection1" file_count:2965 delete_count:10 deleted_byte_count:2639254 replica_placement:100 version:3 modified_at_second:1615601870 + volume id:227 size:1106699864 collection:"collection1" file_count:2827 delete_count:19 deleted_byte_count:5393310 replica_placement:100 version:3 modified_at_second:1615609989 + volume id:228 size:1109784072 collection:"collection1" file_count:2504 delete_count:24 deleted_byte_count:5458950 replica_placement:100 version:3 modified_at_second:1615610489 + volume id:229 size:1109855256 collection:"collection1" file_count:2857 delete_count:22 deleted_byte_count:2839883 replica_placement:100 version:3 modified_at_second:1615609989 + volume id:231 size:1112917664 collection:"collection1" file_count:3151 delete_count:19 deleted_byte_count:2852517 replica_placement:100 version:3 modified_at_second:1615611350 + volume id:232 size:1073901520 collection:"collection1" file_count:3004 delete_count:54 deleted_byte_count:10273081 replica_placement:100 version:3 modified_at_second:1615611352 + volume id:233 size:1080526464 collection:"collection1" file_count:3136 delete_count:61 deleted_byte_count:17991717 replica_placement:100 version:3 modified_at_second:1615611354 + volume id:236 size:1089476200 collection:"collection1" file_count:3231 delete_count:53 deleted_byte_count:11625921 replica_placement:100 version:3 modified_at_second:1615611351 + volume id:238 size:354320000 collection:"collection1" file_count:701 delete_count:17 deleted_byte_count:5940420 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632030 + volume id:240 size:424791528 collection:"collection1" file_count:734 delete_count:12 deleted_byte_count:7353071 replica_placement:100 version:3 modified_at_second:1615631669 + volume id:242 size:1075383304 collection:"collection2" file_count:10470 replica_placement:100 version:3 modified_at_second:1614852115 + volume id:243 size:1088174560 collection:"collection2" file_count:11109 delete_count:1 deleted_byte_count:938 replica_placement:100 version:3 modified_at_second:1614852202 + volume id:245 size:1074597056 collection:"collection2" file_count:10371 delete_count:3 deleted_byte_count:209701 replica_placement:100 version:3 modified_at_second:1614852093 + volume id:247 size:1075859784 collection:"collection2" file_count:10443 delete_count:2 deleted_byte_count:564486 replica_placement:100 version:3 modified_at_second:1614856152 + volume id:249 size:1074819168 collection:"collection2" file_count:10763 delete_count:2 deleted_byte_count:271699 replica_placement:100 version:3 modified_at_second:1614856231 + volume id:250 size:1080572256 collection:"collection2" file_count:10220 replica_placement:100 version:3 modified_at_second:1614856129 + volume id:251 size:1075684408 collection:"collection2" file_count:10847 replica_placement:100 version:3 modified_at_second:1614856270 + volume id:254 size:1074830800 collection:"collection2" file_count:14140 delete_count:2 deleted_byte_count:105892 replica_placement:100 version:3 modified_at_second:1614861115 + volume id:257 size:1082621664 collection:"collection2" file_count:18172 delete_count:2 deleted_byte_count:25125 replica_placement:100 version:3 modified_at_second:1614866395 + volume id:260 size:1075105664 collection:"collection2" file_count:17316 delete_count:4 deleted_byte_count:2015310 replica_placement:100 version:3 modified_at_second:1614866226 + volume id:261 size:1076628592 collection:"collection2" file_count:18355 delete_count:1 deleted_byte_count:1155 replica_placement:100 version:3 modified_at_second:1614866420 + volume id:262 size:1078492464 collection:"collection2" file_count:20390 delete_count:3 deleted_byte_count:287601 replica_placement:100 version:3 modified_at_second:1614871601 + volume id:263 size:1077167440 collection:"collection2" file_count:20227 delete_count:4 deleted_byte_count:97887 replica_placement:100 version:3 modified_at_second:1614871567 + volume id:268 size:1074490592 collection:"collection2" file_count:21698 delete_count:1 deleted_byte_count:33968 replica_placement:100 version:3 modified_at_second:1614877435 + volume id:269 size:1077552720 collection:"collection2" file_count:21875 delete_count:4 deleted_byte_count:347272 replica_placement:100 version:3 modified_at_second:1614877481 + volume id:271 size:1076992648 collection:"collection2" file_count:22640 delete_count:1 deleted_byte_count:30645 replica_placement:100 version:3 modified_at_second:1614877504 + volume id:273 size:1074873432 collection:"collection2" file_count:20511 delete_count:3 deleted_byte_count:46076 replica_placement:100 version:3 modified_at_second:1614884046 + volume id:274 size:1075994128 collection:"collection2" file_count:20997 replica_placement:100 version:3 modified_at_second:1614884113 + volume id:276 size:1076899888 collection:"collection2" file_count:20190 delete_count:1 deleted_byte_count:8798 replica_placement:100 version:3 modified_at_second:1614884003 + volume id:277 size:1074956160 collection:"collection2" file_count:19260 delete_count:2 deleted_byte_count:172356 replica_placement:100 version:3 modified_at_second:1614889988 + volume id:279 size:1077325096 collection:"collection2" file_count:19671 delete_count:6 deleted_byte_count:379116 replica_placement:100 version:3 modified_at_second:1614890230 + volume id:282 size:1075232240 collection:"collection2" file_count:22659 delete_count:4 deleted_byte_count:67915 replica_placement:100 version:3 modified_at_second:1614897304 + volume id:284 size:1074533384 collection:"collection2" file_count:22196 delete_count:4 deleted_byte_count:154683 replica_placement:100 version:3 modified_at_second:1614897231 + volume id:285 size:1082128576 collection:"collection2" file_count:21804 delete_count:1 deleted_byte_count:1064 replica_placement:100 version:3 modified_at_second:1614897165 + volume id:286 size:1077464824 collection:"collection2" file_count:23905 delete_count:6 deleted_byte_count:630577 replica_placement:100 version:3 modified_at_second:1614897401 + volume id:287 size:1074590528 collection:"collection2" file_count:28163 delete_count:5 deleted_byte_count:35727 replica_placement:100 version:3 modified_at_second:1614904874 + volume id:288 size:1075406800 collection:"collection2" file_count:27243 delete_count:2 deleted_byte_count:51519 replica_placement:100 version:3 modified_at_second:1614904738 + volume id:292 size:1092010744 collection:"collection2" file_count:26781 delete_count:5 deleted_byte_count:508910 replica_placement:100 version:3 modified_at_second:1614912327 + volume id:293 size:1075409776 collection:"collection2" file_count:26063 delete_count:4 deleted_byte_count:183834 replica_placement:100 version:3 modified_at_second:1614912235 + volume id:294 size:1075443992 collection:"collection2" file_count:26076 delete_count:4 deleted_byte_count:194914 replica_placement:100 version:3 modified_at_second:1614912220 + volume id:295 size:1074702376 collection:"collection2" file_count:24488 delete_count:3 deleted_byte_count:48555 replica_placement:100 version:3 modified_at_second:1614911929 + volume id:300 size:1076212424 collection:"collection2" file_count:22892 delete_count:2 deleted_byte_count:61320 replica_placement:100 version:3 modified_at_second:1614918464 + volume id:304 size:1081038888 collection:"collection2" file_count:24505 delete_count:2 deleted_byte_count:124447 replica_placement:100 version:3 modified_at_second:1614925567 + volume id:305 size:1074185552 collection:"collection2" file_count:22074 delete_count:5 deleted_byte_count:20221 replica_placement:100 version:3 modified_at_second:1614925312 + volume id:310 size:1074761520 collection:"collection2" file_count:21441 delete_count:3 deleted_byte_count:13934 replica_placement:100 version:3 modified_at_second:1614931077 + volume id:311 size:1088248208 collection:"collection2" file_count:23553 delete_count:6 deleted_byte_count:191716 replica_placement:100 version:3 modified_at_second:1614931460 + volume id:312 size:1075037808 collection:"collection2" file_count:22524 replica_placement:100 version:3 modified_at_second:1614937832 + volume id:313 size:1074876016 collection:"collection2" file_count:22404 delete_count:4 deleted_byte_count:51728 replica_placement:100 version:3 modified_at_second:1614937755 + volume id:314 size:1074670840 collection:"collection2" file_count:20964 delete_count:4 deleted_byte_count:304291 replica_placement:100 version:3 modified_at_second:1614937441 + volume id:315 size:1084153456 collection:"collection2" file_count:23638 delete_count:2 deleted_byte_count:53956 replica_placement:100 version:3 modified_at_second:1614937884 + volume id:316 size:1077720784 collection:"collection2" file_count:22605 delete_count:1 deleted_byte_count:8503 replica_placement:100 version:3 modified_at_second:1614937838 + volume id:317 size:1076215040 collection:"collection2" file_count:23572 delete_count:2 deleted_byte_count:1441356 replica_placement:100 version:3 modified_at_second:1614943965 + volume id:319 size:1073952744 collection:"collection2" file_count:22286 delete_count:2 deleted_byte_count:43421 replica_placement:100 version:3 modified_at_second:1614943810 + volume id:320 size:1082437736 collection:"collection2" file_count:21544 delete_count:3 deleted_byte_count:16712 replica_placement:100 version:3 modified_at_second:1614943591 + volume id:321 size:1081477960 collection:"collection2" file_count:23531 delete_count:5 deleted_byte_count:262564 replica_placement:100 version:3 modified_at_second:1614943982 + volume id:322 size:1078471600 collection:"collection2" file_count:21905 delete_count:3 deleted_byte_count:145002 replica_placement:100 version:3 modified_at_second:1614950574 + volume id:324 size:1075606712 collection:"collection2" file_count:20799 delete_count:1 deleted_byte_count:251210 replica_placement:100 version:3 modified_at_second:1614950310 + volume id:326 size:1076059936 collection:"collection2" file_count:22564 delete_count:2 deleted_byte_count:192886 replica_placement:100 version:3 modified_at_second:1614950619 + volume id:327 size:1076121224 collection:"collection2" file_count:22007 delete_count:3 deleted_byte_count:60358 replica_placement:100 version:3 modified_at_second:1614956487 + volume id:328 size:1074767928 collection:"collection2" file_count:21720 delete_count:3 deleted_byte_count:56429 replica_placement:100 version:3 modified_at_second:1614956362 + volume id:329 size:1076691776 collection:"collection2" file_count:22411 delete_count:5 deleted_byte_count:214092 replica_placement:100 version:3 modified_at_second:1614956485 + volume id:331 size:1074957192 collection:"collection2" file_count:21230 delete_count:4 deleted_byte_count:62145 replica_placement:100 version:3 modified_at_second:1614956259 + volume id:333 size:1074270192 collection:"collection2" file_count:21271 delete_count:2 deleted_byte_count:168122 replica_placement:100 version:3 modified_at_second:1614962697 + volume id:335 size:1076235176 collection:"collection2" file_count:22391 delete_count:3 deleted_byte_count:8838 replica_placement:100 version:3 modified_at_second:1614962970 + volume id:336 size:1087853032 collection:"collection2" file_count:22801 delete_count:2 deleted_byte_count:26394 replica_placement:100 version:3 modified_at_second:1614963003 + volume id:338 size:1076118360 collection:"collection2" file_count:21680 replica_placement:100 version:3 modified_at_second:1614969850 + volume id:342 size:1080186296 collection:"collection2" file_count:22405 delete_count:4 deleted_byte_count:64819 replica_placement:100 version:3 modified_at_second:1614977518 + volume id:343 size:1075345184 collection:"collection2" file_count:21095 delete_count:2 deleted_byte_count:20581 replica_placement:100 version:3 modified_at_second:1614977148 + volume id:349 size:1075957824 collection:"collection2" file_count:22395 delete_count:2 deleted_byte_count:61565 replica_placement:100 version:3 modified_at_second:1614984748 + volume id:350 size:1074756688 collection:"collection2" file_count:21990 delete_count:3 deleted_byte_count:233881 replica_placement:100 version:3 modified_at_second:1614984682 + volume id:354 size:1085213992 collection:"collection2" file_count:23150 delete_count:4 deleted_byte_count:82391 replica_placement:100 version:3 modified_at_second:1614992207 + volume id:356 size:1083304992 collection:"collection2" file_count:21552 delete_count:4 deleted_byte_count:14472 replica_placement:100 version:3 modified_at_second:1614992027 + volume id:358 size:1085152312 collection:"collection2" file_count:23756 delete_count:3 deleted_byte_count:44531 replica_placement:100 version:3 modified_at_second:1614998824 + volume id:359 size:1074211240 collection:"collection2" file_count:22437 delete_count:2 deleted_byte_count:187953 replica_placement:100 version:3 modified_at_second:1614998711 + volume id:362 size:1074074120 collection:"collection2" file_count:20595 delete_count:1 deleted_byte_count:112145 replica_placement:100 version:3 modified_at_second:1615004407 + volume id:363 size:1078859496 collection:"collection2" file_count:23177 delete_count:4 deleted_byte_count:9601 replica_placement:100 version:3 modified_at_second:1615004822 + volume id:364 size:1081280816 collection:"collection2" file_count:22686 delete_count:1 deleted_byte_count:84375 replica_placement:100 version:3 modified_at_second:1615004813 + volume id:365 size:1075736632 collection:"collection2" file_count:22193 delete_count:5 deleted_byte_count:259033 replica_placement:100 version:3 modified_at_second:1615004776 + volume id:366 size:1075267272 collection:"collection2" file_count:21856 delete_count:5 deleted_byte_count:138363 replica_placement:100 version:3 modified_at_second:1615004703 + volume id:367 size:1076403648 collection:"collection2" file_count:22995 delete_count:2 deleted_byte_count:36955 replica_placement:100 version:3 modified_at_second:1615010985 + volume id:368 size:1074822016 collection:"collection2" file_count:22252 delete_count:4 deleted_byte_count:3291946 replica_placement:100 version:3 modified_at_second:1615010878 + volume id:369 size:1091472040 collection:"collection2" file_count:23709 delete_count:4 deleted_byte_count:400876 replica_placement:100 version:3 modified_at_second:1615011019 + volume id:370 size:1076040480 collection:"collection2" file_count:22092 delete_count:2 deleted_byte_count:115388 replica_placement:100 version:3 modified_at_second:1615010877 + volume id:371 size:1078806160 collection:"collection2" file_count:22685 delete_count:2 deleted_byte_count:68905 replica_placement:100 version:3 modified_at_second:1615010994 + volume id:372 size:1076193312 collection:"collection2" file_count:22774 delete_count:1 deleted_byte_count:3495 replica_placement:100 version:3 modified_at_second:1615016911 + volume id:374 size:1085011080 collection:"collection2" file_count:23054 delete_count:2 deleted_byte_count:89034 replica_placement:100 version:3 modified_at_second:1615016917 + volume id:375 size:1076140688 collection:"collection2" file_count:21880 delete_count:2 deleted_byte_count:51103 replica_placement:100 version:3 modified_at_second:1615016787 + volume id:378 size:959273824 collection:"collection2" file_count:15031 replica_placement:100 version:3 modified_at_second:1615632323 + volume id:379 size:1014108592 collection:"collection2" file_count:15360 delete_count:8 deleted_byte_count:2524591 replica_placement:100 version:3 modified_at_second:1615632323 + volume id:380 size:1010760464 collection:"collection2" file_count:14920 delete_count:7 deleted_byte_count:134859 replica_placement:100 version:3 modified_at_second:1615632322 + Disk hdd total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457 + DataNode 192.168.1.1:8080 total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457 + Rack DefaultRack total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457 + DataCenter dc4 total size:274627838960 file_count:3607097 deleted_file:13594 deleted_bytes:4185524457 + DataCenter dc5 hdd(volume:299/3000 active:299 free:2701 remote:0) + Rack DefaultRack hdd(volume:299/3000 active:299 free:2701 remote:0) + DataNode 192.168.1.5:8080 hdd(volume:299/3000 active:299 free:2701 remote:0) + Disk hdd(volume:299/3000 active:299 free:2701 remote:0) + volume id:5 size:293806008 file_count:1669 delete_count:2 deleted_byte_count:274334 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631342 + volume id:11 size:1109224552 collection:"collection0" file_count:44 replica_placement:100 version:3 modified_at_second:1615629606 + volume id:18 size:1096678688 collection:"collection0" file_count:88 delete_count:4 deleted_byte_count:8633 replica_placement:100 version:3 modified_at_second:1615631673 + volume id:19 size:1096923792 collection:"collection0" file_count:100 delete_count:10 deleted_byte_count:75779917 replica_placement:100 version:3 compact_revision:4 modified_at_second:1615630117 + volume id:20 size:1074760432 collection:"collection0" file_count:82 delete_count:5 deleted_byte_count:12156 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615629340 + volume id:26 size:532657632 collection:"collection0" file_count:100 delete_count:4 deleted_byte_count:9081 replica_placement:100 version:3 modified_at_second:1614170024 + volume id:27 size:298886792 file_count:1608 replica_placement:100 version:3 modified_at_second:1615632482 + volume id:28 size:308919192 file_count:1591 delete_count:1 deleted_byte_count:125280 replica_placement:100 version:3 modified_at_second:1615631762 + volume id:29 size:281582688 file_count:1537 replica_placement:100 version:3 modified_at_second:1615629422 + volume id:30 size:289466144 file_count:1566 delete_count:1 deleted_byte_count:124972 replica_placement:100 version:3 modified_at_second:1615632422 + volume id:31 size:273363256 file_count:1498 replica_placement:100 version:3 modified_at_second:1615631642 + volume id:32 size:281343360 file_count:1497 replica_placement:100 version:3 modified_at_second:1615632362 + volume id:33 size:1130226344 collection:"collection1" file_count:7322 delete_count:172 deleted_byte_count:45199399 replica_placement:100 version:3 modified_at_second:1615618789 + volume id:34 size:1077111136 collection:"collection1" file_count:9781 delete_count:110 deleted_byte_count:20894827 replica_placement:100 version:3 modified_at_second:1615619366 + volume id:35 size:1075241744 collection:"collection1" file_count:10523 delete_count:97 deleted_byte_count:46586217 replica_placement:100 version:3 modified_at_second:1615618790 + volume id:36 size:1075118336 collection:"collection1" file_count:10341 delete_count:118 deleted_byte_count:24753278 replica_placement:100 version:3 modified_at_second:1615606148 + volume id:37 size:1075895576 collection:"collection1" file_count:12013 delete_count:98 deleted_byte_count:50747932 replica_placement:100 version:3 modified_at_second:1615594776 + volume id:38 size:1075545744 collection:"collection1" file_count:13324 delete_count:100 deleted_byte_count:25223906 replica_placement:100 version:3 modified_at_second:1615569830 + volume id:39 size:1076606536 collection:"collection1" file_count:12612 delete_count:78 deleted_byte_count:17462730 replica_placement:100 version:3 modified_at_second:1615611959 + volume id:40 size:1075358552 collection:"collection1" file_count:12597 delete_count:62 deleted_byte_count:11657901 replica_placement:100 version:3 modified_at_second:1615612994 + volume id:41 size:1076283592 collection:"collection1" file_count:12088 delete_count:84 deleted_byte_count:19311237 replica_placement:100 version:3 modified_at_second:1615596736 + volume id:42 size:1093948352 collection:"collection1" file_count:7889 delete_count:47 deleted_byte_count:5697275 replica_placement:100 version:3 modified_at_second:1615548906 + volume id:43 size:1116445864 collection:"collection1" file_count:7355 delete_count:57 deleted_byte_count:9727158 replica_placement:100 version:3 modified_at_second:1615566167 + volume id:44 size:1077582560 collection:"collection1" file_count:7295 delete_count:50 deleted_byte_count:12618414 replica_placement:100 version:3 modified_at_second:1615566170 + volume id:45 size:1075254640 collection:"collection1" file_count:10772 delete_count:76 deleted_byte_count:22426345 replica_placement:100 version:3 modified_at_second:1615573498 + volume id:46 size:1075286056 collection:"collection1" file_count:9947 delete_count:309 deleted_byte_count:105601163 replica_placement:100 version:3 modified_at_second:1615569825 + volume id:47 size:444599784 collection:"collection1" file_count:709 delete_count:19 deleted_byte_count:11913451 replica_placement:100 version:3 modified_at_second:1615632397 + volume id:48 size:1076778664 collection:"collection1" file_count:9850 delete_count:77 deleted_byte_count:16641907 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630690 + volume id:49 size:1078775288 collection:"collection1" file_count:9631 delete_count:27 deleted_byte_count:5985628 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615575823 + volume id:50 size:1076688288 collection:"collection1" file_count:7921 delete_count:26 deleted_byte_count:5162032 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615610876 + volume id:51 size:1076796120 collection:"collection1" file_count:10550 delete_count:39 deleted_byte_count:12723654 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615547786 + volume id:53 size:1063089216 collection:"collection1" file_count:9832 delete_count:31 deleted_byte_count:9273066 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632156 + volume id:54 size:1045022288 collection:"collection1" file_count:9409 delete_count:29 deleted_byte_count:15102818 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630813 + volume id:55 size:1012890016 collection:"collection1" file_count:8651 delete_count:27 deleted_byte_count:9418841 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631453 + volume id:56 size:1002412240 collection:"collection1" file_count:8762 delete_count:40 deleted_byte_count:65885831 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632014 + volume id:57 size:839849792 collection:"collection1" file_count:7514 delete_count:24 deleted_byte_count:6228543 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631775 + volume id:58 size:908064192 collection:"collection1" file_count:8128 delete_count:21 deleted_byte_count:6113731 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632343 + volume id:59 size:988302272 collection:"collection1" file_count:8098 delete_count:20 deleted_byte_count:3947615 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632238 + volume id:60 size:1010702480 collection:"collection1" file_count:8969 delete_count:79 deleted_byte_count:24782814 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632439 + volume id:61 size:975604544 collection:"collection1" file_count:8683 delete_count:20 deleted_byte_count:10276072 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615631176 + volume id:62 size:873845904 collection:"collection1" file_count:7897 delete_count:23 deleted_byte_count:10920170 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631133 + volume id:63 size:956941176 collection:"collection1" file_count:8271 delete_count:32 deleted_byte_count:15876189 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632036 + volume id:64 size:965638424 collection:"collection1" file_count:8218 delete_count:27 deleted_byte_count:6922489 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631032 + volume id:65 size:823283608 collection:"collection1" file_count:7834 delete_count:29 deleted_byte_count:5950610 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632307 + volume id:66 size:821343440 collection:"collection1" file_count:7383 delete_count:29 deleted_byte_count:12010343 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631968 + volume id:67 size:878713880 collection:"collection1" file_count:7299 delete_count:117 deleted_byte_count:24857326 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632156 + volume id:69 size:863913896 collection:"collection1" file_count:7291 delete_count:100 deleted_byte_count:25335024 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630534 + volume id:70 size:886695472 collection:"collection1" file_count:7769 delete_count:164 deleted_byte_count:45162513 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632398 + volume id:71 size:907608392 collection:"collection1" file_count:7658 delete_count:122 deleted_byte_count:27622941 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632307 + volume id:72 size:903990720 collection:"collection1" file_count:6996 delete_count:240 deleted_byte_count:74147727 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630985 + volume id:73 size:929047544 collection:"collection1" file_count:7038 delete_count:227 deleted_byte_count:65336664 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630707 + volume id:75 size:908045000 collection:"collection1" file_count:6911 delete_count:268 deleted_byte_count:73934373 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632430 + volume id:76 size:985296744 collection:"collection1" file_count:6566 delete_count:61 deleted_byte_count:44464430 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632284 + volume id:77 size:929398296 collection:"collection1" file_count:7427 delete_count:238 deleted_byte_count:59581579 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632014 + volume id:78 size:1075671512 collection:"collection1" file_count:7540 delete_count:258 deleted_byte_count:71726846 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615582829 + volume id:79 size:948225472 collection:"collection1" file_count:6997 delete_count:227 deleted_byte_count:60625763 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631326 + volume id:80 size:827912928 collection:"collection1" file_count:6916 delete_count:15 deleted_byte_count:5611604 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631159 + volume id:81 size:880693168 collection:"collection1" file_count:7481 delete_count:238 deleted_byte_count:80880878 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631395 + volume id:82 size:1041660512 collection:"collection1" file_count:7043 delete_count:207 deleted_byte_count:52275724 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632430 + volume id:83 size:936194288 collection:"collection1" file_count:7593 delete_count:13 deleted_byte_count:4633917 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632029 + volume id:84 size:871262320 collection:"collection1" file_count:8190 delete_count:14 deleted_byte_count:3150948 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631161 + volume id:86 size:1009434632 collection:"collection1" file_count:8474 delete_count:236 deleted_byte_count:64543674 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630812 + volume id:87 size:922274624 collection:"collection1" file_count:12902 delete_count:13 deleted_byte_count:3412959 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615632438 + volume id:88 size:1073767976 collection:"collection1" file_count:14994 delete_count:207 deleted_byte_count:82380696 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615541383 + volume id:89 size:1044421824 collection:"collection1" file_count:14943 delete_count:243 deleted_byte_count:58543159 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615632208 + volume id:90 size:891163760 collection:"collection1" file_count:14608 delete_count:10 deleted_byte_count:2564369 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615629392 + volume id:91 size:936573952 collection:"collection1" file_count:14686 delete_count:11 deleted_byte_count:4717727 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631851 + volume id:92 size:992439144 collection:"collection1" file_count:7061 delete_count:195 deleted_byte_count:60649573 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630566 + volume id:93 size:1079602592 collection:"collection1" file_count:7878 delete_count:270 deleted_byte_count:74150048 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615556013 + volume id:94 size:1030684704 collection:"collection1" file_count:7660 delete_count:207 deleted_byte_count:70150733 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631616 + volume id:95 size:990877824 collection:"collection1" file_count:6620 delete_count:206 deleted_byte_count:60363604 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615631867 + volume id:96 size:989294848 collection:"collection1" file_count:7544 delete_count:229 deleted_byte_count:59931853 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630778 + volume id:98 size:1077836472 collection:"collection1" file_count:7605 delete_count:202 deleted_byte_count:73180379 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615523691 + volume id:99 size:1071718496 collection:"collection1" file_count:7470 delete_count:8 deleted_byte_count:9624950 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615631175 + volume id:100 size:1083617472 collection:"collection1" file_count:7018 delete_count:187 deleted_byte_count:61304236 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615505914 + volume id:101 size:1077109408 collection:"collection1" file_count:7706 delete_count:226 deleted_byte_count:77864780 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615630994 + volume id:102 size:1074359920 collection:"collection1" file_count:7338 delete_count:7 deleted_byte_count:6499151 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615626682 + volume id:103 size:1075863904 collection:"collection1" file_count:7184 delete_count:186 deleted_byte_count:58872238 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615628417 + volume id:106 size:1075458680 collection:"collection1" file_count:7182 delete_count:307 deleted_byte_count:69349053 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615598137 + volume id:107 size:1073811776 collection:"collection1" file_count:7436 delete_count:168 deleted_byte_count:57747428 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615293569 + volume id:108 size:1074648024 collection:"collection1" file_count:7472 delete_count:194 deleted_byte_count:70864699 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615593231 + volume id:109 size:1075254560 collection:"collection1" file_count:7556 delete_count:263 deleted_byte_count:55155265 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615502487 + volume id:110 size:1076575744 collection:"collection1" file_count:6996 delete_count:163 deleted_byte_count:52954032 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615590786 + volume id:111 size:1073826176 collection:"collection1" file_count:7355 delete_count:155 deleted_byte_count:50083578 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615593232 + volume id:112 size:1076392512 collection:"collection1" file_count:8291 delete_count:156 deleted_byte_count:74120183 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615569823 + volume id:113 size:1076709184 collection:"collection1" file_count:9355 delete_count:177 deleted_byte_count:59796765 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615569822 + volume id:114 size:1074762792 collection:"collection1" file_count:8802 delete_count:156 deleted_byte_count:38470055 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615591826 + volume id:115 size:1076192296 collection:"collection1" file_count:7690 delete_count:154 deleted_byte_count:32267193 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615285296 + volume id:117 size:1073917192 collection:"collection1" file_count:9520 delete_count:114 deleted_byte_count:21835126 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615573712 + volume id:118 size:1074064344 collection:"collection1" file_count:8738 delete_count:15 deleted_byte_count:3460697 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615516264 + volume id:120 size:1076115928 collection:"collection1" file_count:9639 delete_count:118 deleted_byte_count:33357871 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615482567 + volume id:121 size:1078803320 collection:"collection1" file_count:10113 delete_count:441 deleted_byte_count:94128627 replica_placement:100 version:3 modified_at_second:1615506626 + volume id:122 size:1076235312 collection:"collection1" file_count:9106 delete_count:252 deleted_byte_count:93041272 replica_placement:100 version:3 modified_at_second:1615585912 + volume id:123 size:1080491112 collection:"collection1" file_count:10623 delete_count:302 deleted_byte_count:83956998 replica_placement:100 version:3 modified_at_second:1615585916 + volume id:124 size:1074519360 collection:"collection1" file_count:9457 delete_count:286 deleted_byte_count:74752459 replica_placement:100 version:3 modified_at_second:1615585913 + volume id:125 size:1088687040 collection:"collection1" file_count:9518 delete_count:281 deleted_byte_count:76037905 replica_placement:100 version:3 modified_at_second:1615585913 + volume id:126 size:1073867408 collection:"collection1" file_count:9320 delete_count:278 deleted_byte_count:94547424 replica_placement:100 version:3 modified_at_second:1615585911 + volume id:127 size:1074907336 collection:"collection1" file_count:9900 delete_count:133 deleted_byte_count:48570820 replica_placement:100 version:3 modified_at_second:1615612990 + volume id:128 size:1074874632 collection:"collection1" file_count:9821 delete_count:148 deleted_byte_count:43633334 replica_placement:100 version:3 modified_at_second:1615602670 + volume id:129 size:1074704328 collection:"collection1" file_count:10012 delete_count:150 deleted_byte_count:64491721 replica_placement:100 version:3 modified_at_second:1615627566 + volume id:130 size:1075000632 collection:"collection1" file_count:10633 delete_count:161 deleted_byte_count:34768201 replica_placement:100 version:3 modified_at_second:1615582327 + volume id:131 size:1075279584 collection:"collection1" file_count:10075 delete_count:135 deleted_byte_count:29795712 replica_placement:100 version:3 modified_at_second:1615523898 + volume id:132 size:1088539552 collection:"collection1" file_count:11051 delete_count:71 deleted_byte_count:17178322 replica_placement:100 version:3 modified_at_second:1615619581 + volume id:134 size:1074367304 collection:"collection1" file_count:10662 delete_count:69 deleted_byte_count:25530139 replica_placement:100 version:3 modified_at_second:1615555873 + volume id:135 size:1073906776 collection:"collection1" file_count:10446 delete_count:71 deleted_byte_count:28599975 replica_placement:100 version:3 modified_at_second:1615569816 + volume id:136 size:1074433552 collection:"collection1" file_count:9593 delete_count:72 deleted_byte_count:26912512 replica_placement:100 version:3 modified_at_second:1615376036 + volume id:137 size:1074309264 collection:"collection1" file_count:9633 delete_count:50 deleted_byte_count:27487972 replica_placement:100 version:3 modified_at_second:1615572231 + volume id:138 size:1074465744 collection:"collection1" file_count:10120 delete_count:55 deleted_byte_count:15875438 replica_placement:100 version:3 modified_at_second:1615572231 + volume id:140 size:1076203744 collection:"collection1" file_count:11219 delete_count:57 deleted_byte_count:19864498 replica_placement:100 version:3 modified_at_second:1615571947 + volume id:141 size:1074619488 collection:"collection1" file_count:9840 delete_count:45 deleted_byte_count:40890181 replica_placement:100 version:3 modified_at_second:1615630994 + volume id:142 size:1075733064 collection:"collection1" file_count:9009 delete_count:48 deleted_byte_count:9912854 replica_placement:100 version:3 modified_at_second:1615598913 + volume id:143 size:1075011280 collection:"collection1" file_count:9608 delete_count:51 deleted_byte_count:37282460 replica_placement:100 version:3 modified_at_second:1615488584 + volume id:144 size:1074549720 collection:"collection1" file_count:8780 delete_count:50 deleted_byte_count:52475146 replica_placement:100 version:3 modified_at_second:1615573451 + volume id:145 size:1074394928 collection:"collection1" file_count:9255 delete_count:34 deleted_byte_count:38011392 replica_placement:100 version:3 modified_at_second:1615591825 + volume id:146 size:1076337576 collection:"collection1" file_count:10492 delete_count:50 deleted_byte_count:17071505 replica_placement:100 version:3 modified_at_second:1615632005 + volume id:147 size:1077130576 collection:"collection1" file_count:10451 delete_count:27 deleted_byte_count:8290907 replica_placement:100 version:3 modified_at_second:1615604115 + volume id:148 size:1076066568 collection:"collection1" file_count:9547 delete_count:33 deleted_byte_count:7034089 replica_placement:100 version:3 modified_at_second:1615586390 + volume id:149 size:1074989016 collection:"collection1" file_count:8352 delete_count:35 deleted_byte_count:7179742 replica_placement:100 version:3 modified_at_second:1615494494 + volume id:150 size:1076290328 collection:"collection1" file_count:9328 delete_count:33 deleted_byte_count:43417791 replica_placement:100 version:3 modified_at_second:1615611567 + volume id:152 size:1075941400 collection:"collection1" file_count:9951 delete_count:36 deleted_byte_count:25348335 replica_placement:100 version:3 modified_at_second:1615606614 + volume id:153 size:1078539784 collection:"collection1" file_count:10924 delete_count:34 deleted_byte_count:12603081 replica_placement:100 version:3 modified_at_second:1615606614 + volume id:154 size:1081244696 collection:"collection1" file_count:11002 delete_count:31 deleted_byte_count:8467560 replica_placement:100 version:3 modified_at_second:1615478469 + volume id:155 size:1075140688 collection:"collection1" file_count:10882 delete_count:32 deleted_byte_count:10076804 replica_placement:100 version:3 modified_at_second:1615606614 + volume id:156 size:1074975832 collection:"collection1" file_count:9535 delete_count:40 deleted_byte_count:11426621 replica_placement:100 version:3 modified_at_second:1615628341 + volume id:157 size:1076758536 collection:"collection1" file_count:10012 delete_count:19 deleted_byte_count:11688737 replica_placement:100 version:3 modified_at_second:1615597782 + volume id:158 size:1087251976 collection:"collection1" file_count:9972 delete_count:20 deleted_byte_count:10328429 replica_placement:100 version:3 modified_at_second:1615588879 + volume id:159 size:1074132368 collection:"collection1" file_count:9382 delete_count:27 deleted_byte_count:11474574 replica_placement:100 version:3 modified_at_second:1615593593 + volume id:160 size:1075680952 collection:"collection1" file_count:9772 delete_count:22 deleted_byte_count:4981968 replica_placement:100 version:3 modified_at_second:1615597780 + volume id:162 size:1074286880 collection:"collection1" file_count:11220 delete_count:17 deleted_byte_count:1815547 replica_placement:100 version:3 modified_at_second:1615478126 + volume id:163 size:1074457192 collection:"collection1" file_count:12524 delete_count:27 deleted_byte_count:6359619 replica_placement:100 version:3 modified_at_second:1615579313 + volume id:164 size:1074261248 collection:"collection1" file_count:11922 delete_count:25 deleted_byte_count:2923288 replica_placement:100 version:3 modified_at_second:1615620084 + volume id:165 size:1073891016 collection:"collection1" file_count:9152 delete_count:12 deleted_byte_count:19164659 replica_placement:100 version:3 modified_at_second:1615471907 + volume id:166 size:1075637536 collection:"collection1" file_count:14211 delete_count:24 deleted_byte_count:20415490 replica_placement:100 version:3 modified_at_second:1615491019 + volume id:168 size:1074718808 collection:"collection1" file_count:25702 delete_count:40 deleted_byte_count:4024775 replica_placement:100 version:3 modified_at_second:1615585664 + volume id:169 size:1073863128 collection:"collection1" file_count:25248 delete_count:43 deleted_byte_count:3013817 replica_placement:100 version:3 modified_at_second:1615569832 + volume id:170 size:1075747096 collection:"collection1" file_count:24596 delete_count:41 deleted_byte_count:3494711 replica_placement:100 version:3 modified_at_second:1615579204 + volume id:171 size:1081881312 collection:"collection1" file_count:24215 delete_count:36 deleted_byte_count:3191335 replica_placement:100 version:3 modified_at_second:1615596485 + volume id:172 size:1074787312 collection:"collection1" file_count:31236 delete_count:50 deleted_byte_count:3316482 replica_placement:100 version:3 modified_at_second:1615612385 + volume id:173 size:1074154648 collection:"collection1" file_count:30884 delete_count:34 deleted_byte_count:2430948 replica_placement:100 version:3 modified_at_second:1615591904 + volume id:175 size:1077742504 collection:"collection1" file_count:32353 delete_count:33 deleted_byte_count:1861403 replica_placement:100 version:3 modified_at_second:1615559515 + volume id:176 size:1073854800 collection:"collection1" file_count:30582 delete_count:34 deleted_byte_count:7701976 replica_placement:100 version:3 modified_at_second:1615626169 + volume id:177 size:1074120120 collection:"collection1" file_count:22293 delete_count:16 deleted_byte_count:3719562 replica_placement:100 version:3 modified_at_second:1615516891 + volume id:178 size:1087560112 collection:"collection1" file_count:23482 delete_count:22 deleted_byte_count:18810492 replica_placement:100 version:3 modified_at_second:1615541369 + volume id:180 size:1078438536 collection:"collection1" file_count:23614 delete_count:12 deleted_byte_count:4496474 replica_placement:100 version:3 modified_at_second:1614773242 + volume id:181 size:1074571768 collection:"collection1" file_count:22898 delete_count:19 deleted_byte_count:6628413 replica_placement:100 version:3 modified_at_second:1614745116 + volume id:182 size:1076131280 collection:"collection1" file_count:31987 delete_count:21 deleted_byte_count:1416142 replica_placement:100 version:3 modified_at_second:1615568922 + volume id:183 size:1076361448 collection:"collection1" file_count:31293 delete_count:16 deleted_byte_count:468841 replica_placement:100 version:3 modified_at_second:1615572982 + volume id:184 size:1074594160 collection:"collection1" file_count:31368 delete_count:22 deleted_byte_count:857453 replica_placement:100 version:3 modified_at_second:1615586578 + volume id:185 size:1074099624 collection:"collection1" file_count:30612 delete_count:17 deleted_byte_count:2610847 replica_placement:100 version:3 modified_at_second:1615506832 + volume id:186 size:1074220864 collection:"collection1" file_count:31450 delete_count:15 deleted_byte_count:391855 replica_placement:100 version:3 modified_at_second:1615614933 + volume id:187 size:1074395944 collection:"collection1" file_count:31853 delete_count:17 deleted_byte_count:454283 replica_placement:100 version:3 modified_at_second:1615590490 + volume id:188 size:1074732792 collection:"collection1" file_count:31867 delete_count:19 deleted_byte_count:393743 replica_placement:100 version:3 modified_at_second:1615487645 + volume id:189 size:1074847896 collection:"collection1" file_count:31450 delete_count:16 deleted_byte_count:1040552 replica_placement:100 version:3 modified_at_second:1615335661 + volume id:190 size:1074008912 collection:"collection1" file_count:31987 delete_count:11 deleted_byte_count:685125 replica_placement:100 version:3 modified_at_second:1615447161 + volume id:191 size:1075493024 collection:"collection1" file_count:31301 delete_count:19 deleted_byte_count:708401 replica_placement:100 version:3 modified_at_second:1615357456 + volume id:192 size:1075857400 collection:"collection1" file_count:31490 delete_count:25 deleted_byte_count:720617 replica_placement:100 version:3 modified_at_second:1615621632 + volume id:193 size:1076616768 collection:"collection1" file_count:31907 delete_count:16 deleted_byte_count:464900 replica_placement:100 version:3 modified_at_second:1615507875 + volume id:194 size:1073985624 collection:"collection1" file_count:31434 delete_count:18 deleted_byte_count:391432 replica_placement:100 version:3 modified_at_second:1615559499 + volume id:195 size:1074158312 collection:"collection1" file_count:31453 delete_count:15 deleted_byte_count:718266 replica_placement:100 version:3 modified_at_second:1615559331 + volume id:196 size:1074594784 collection:"collection1" file_count:31665 delete_count:18 deleted_byte_count:3468922 replica_placement:100 version:3 modified_at_second:1615501688 + volume id:197 size:1075423296 collection:"collection1" file_count:16473 delete_count:15 deleted_byte_count:12552442 replica_placement:100 version:3 modified_at_second:1615485253 + volume id:198 size:1075104712 collection:"collection1" file_count:16577 delete_count:18 deleted_byte_count:6583181 replica_placement:100 version:3 modified_at_second:1615623369 + volume id:199 size:1078117688 collection:"collection1" file_count:16497 delete_count:14 deleted_byte_count:1514286 replica_placement:100 version:3 modified_at_second:1615585984 + volume id:200 size:1075630536 collection:"collection1" file_count:16380 delete_count:18 deleted_byte_count:1103109 replica_placement:100 version:3 modified_at_second:1615485252 + volume id:201 size:1091460440 collection:"collection1" file_count:16684 delete_count:26 deleted_byte_count:5590335 replica_placement:100 version:3 modified_at_second:1615585987 + volume id:202 size:1077533160 collection:"collection1" file_count:2847 delete_count:67 deleted_byte_count:65172985 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615588497 + volume id:203 size:1027316272 collection:"collection1" file_count:3040 delete_count:11 deleted_byte_count:3993230 replica_placement:100 version:3 compact_revision:3 modified_at_second:1615631728 + volume id:204 size:1079766872 collection:"collection1" file_count:3233 delete_count:255 deleted_byte_count:104707641 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615565701 + volume id:205 size:1078485304 collection:"collection1" file_count:2869 delete_count:43 deleted_byte_count:18290259 replica_placement:100 version:3 compact_revision:2 modified_at_second:1615579314 + volume id:206 size:1082045848 collection:"collection1" file_count:2979 delete_count:225 deleted_byte_count:88220074 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630989 + volume id:207 size:1081939960 collection:"collection1" file_count:3010 delete_count:4 deleted_byte_count:692350 replica_placement:100 version:3 modified_at_second:1615269061 + volume id:208 size:1077863624 collection:"collection1" file_count:3147 delete_count:6 deleted_byte_count:858726 replica_placement:100 version:3 modified_at_second:1615495515 + volume id:210 size:1094311304 collection:"collection1" file_count:3468 delete_count:4 deleted_byte_count:466433 replica_placement:100 version:3 modified_at_second:1615495515 + volume id:212 size:1078293448 collection:"collection1" file_count:3106 delete_count:6 deleted_byte_count:2085755 replica_placement:100 version:3 modified_at_second:1615586387 + volume id:213 size:1093588072 collection:"collection1" file_count:3681 delete_count:12 deleted_byte_count:3138791 replica_placement:100 version:3 modified_at_second:1615586387 + volume id:214 size:1074486992 collection:"collection1" file_count:3217 delete_count:10 deleted_byte_count:6392871 replica_placement:100 version:3 modified_at_second:1615586383 + volume id:215 size:1074798704 collection:"collection1" file_count:2819 delete_count:31 deleted_byte_count:10873569 replica_placement:100 version:3 modified_at_second:1615586386 + volume id:217 size:1075381872 collection:"collection1" file_count:3331 delete_count:14 deleted_byte_count:2009141 replica_placement:100 version:3 modified_at_second:1615401638 + volume id:218 size:1081263944 collection:"collection1" file_count:3433 delete_count:14 deleted_byte_count:3454237 replica_placement:100 version:3 modified_at_second:1615603637 + volume id:219 size:1092298816 collection:"collection1" file_count:3193 delete_count:17 deleted_byte_count:2047576 replica_placement:100 version:3 modified_at_second:1615579316 + volume id:220 size:1081928312 collection:"collection1" file_count:3166 delete_count:13 deleted_byte_count:4127709 replica_placement:100 version:3 modified_at_second:1615579317 + volume id:221 size:1106545456 collection:"collection1" file_count:3153 delete_count:11 deleted_byte_count:1496835 replica_placement:100 version:3 modified_at_second:1615269138 + volume id:222 size:1106623104 collection:"collection1" file_count:3273 delete_count:11 deleted_byte_count:2114627 replica_placement:100 version:3 modified_at_second:1615586243 + volume id:223 size:1075233064 collection:"collection1" file_count:2966 delete_count:9 deleted_byte_count:744001 replica_placement:100 version:3 modified_at_second:1615586244 + volume id:224 size:1093691520 collection:"collection1" file_count:3463 delete_count:10 deleted_byte_count:1128328 replica_placement:100 version:3 modified_at_second:1615601870 + volume id:225 size:1080698928 collection:"collection1" file_count:3115 delete_count:7 deleted_byte_count:18170416 replica_placement:100 version:3 modified_at_second:1615434684 + volume id:226 size:1103504768 collection:"collection1" file_count:2965 delete_count:10 deleted_byte_count:2639254 replica_placement:100 version:3 modified_at_second:1615601867 + volume id:228 size:1109784072 collection:"collection1" file_count:2504 delete_count:24 deleted_byte_count:5458950 replica_placement:100 version:3 modified_at_second:1615610489 + volume id:230 size:1080722984 collection:"collection1" file_count:2898 delete_count:15 deleted_byte_count:3929261 replica_placement:100 version:3 modified_at_second:1615610537 + volume id:232 size:1073901520 collection:"collection1" file_count:3004 delete_count:54 deleted_byte_count:10273081 replica_placement:100 version:3 modified_at_second:1615611351 + volume id:234 size:1073835280 collection:"collection1" file_count:2965 delete_count:41 deleted_byte_count:4960354 replica_placement:100 version:3 modified_at_second:1615611351 + volume id:235 size:1075586104 collection:"collection1" file_count:2767 delete_count:33 deleted_byte_count:3216540 replica_placement:100 version:3 modified_at_second:1615611354 + volume id:236 size:1089476136 collection:"collection1" file_count:3231 delete_count:53 deleted_byte_count:11625921 replica_placement:100 version:3 modified_at_second:1615611351 + volume id:237 size:375722792 collection:"collection1" file_count:736 delete_count:16 deleted_byte_count:4464870 replica_placement:100 version:3 modified_at_second:1615631727 + volume id:238 size:354320000 collection:"collection1" file_count:701 delete_count:17 deleted_byte_count:5940420 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615632030 + volume id:239 size:426569024 collection:"collection1" file_count:693 delete_count:19 deleted_byte_count:13020783 replica_placement:100 version:3 compact_revision:1 modified_at_second:1615630841 + volume id:240 size:424791528 collection:"collection1" file_count:733 delete_count:13 deleted_byte_count:7515220 replica_placement:100 version:3 modified_at_second:1615631670 + volume id:241 size:380217424 collection:"collection1" file_count:633 delete_count:6 deleted_byte_count:1715768 replica_placement:100 version:3 modified_at_second:1615632006 + volume id:242 size:1075383392 collection:"collection2" file_count:10470 replica_placement:100 version:3 modified_at_second:1614852116 + volume id:243 size:1088174704 collection:"collection2" file_count:11109 delete_count:1 deleted_byte_count:938 replica_placement:100 version:3 modified_at_second:1614852203 + volume id:244 size:1080295352 collection:"collection2" file_count:10812 delete_count:1 deleted_byte_count:795 replica_placement:100 version:3 modified_at_second:1615628825 + volume id:246 size:1075998672 collection:"collection2" file_count:10365 delete_count:1 deleted_byte_count:13112 replica_placement:100 version:3 modified_at_second:1614852106 + volume id:247 size:1075859808 collection:"collection2" file_count:10443 delete_count:2 deleted_byte_count:564486 replica_placement:100 version:3 modified_at_second:1614856152 + volume id:248 size:1084301208 collection:"collection2" file_count:11217 delete_count:4 deleted_byte_count:746488 replica_placement:100 version:3 modified_at_second:1614856285 + volume id:250 size:1080572168 collection:"collection2" file_count:10220 replica_placement:100 version:3 modified_at_second:1614856129 + volume id:252 size:1075065264 collection:"collection2" file_count:14622 delete_count:2 deleted_byte_count:5228 replica_placement:100 version:3 modified_at_second:1614861200 + volume id:253 size:1087328880 collection:"collection2" file_count:14920 delete_count:3 deleted_byte_count:522994 replica_placement:100 version:3 modified_at_second:1614861258 + volume id:254 size:1074830736 collection:"collection2" file_count:14140 delete_count:2 deleted_byte_count:105892 replica_placement:100 version:3 modified_at_second:1614861115 + volume id:255 size:1079581640 collection:"collection2" file_count:14877 delete_count:3 deleted_byte_count:101223 replica_placement:100 version:3 modified_at_second:1614861233 + volume id:256 size:1074283592 collection:"collection2" file_count:14157 delete_count:1 deleted_byte_count:18156 replica_placement:100 version:3 modified_at_second:1614861100 + volume id:257 size:1082621720 collection:"collection2" file_count:18172 delete_count:2 deleted_byte_count:25125 replica_placement:100 version:3 modified_at_second:1614866402 + volume id:258 size:1075527216 collection:"collection2" file_count:18421 delete_count:4 deleted_byte_count:267833 replica_placement:100 version:3 modified_at_second:1614866420 + volume id:259 size:1075507848 collection:"collection2" file_count:18079 delete_count:2 deleted_byte_count:71992 replica_placement:100 version:3 modified_at_second:1614866381 + volume id:260 size:1075105664 collection:"collection2" file_count:17316 delete_count:4 deleted_byte_count:2015310 replica_placement:100 version:3 modified_at_second:1614866226 + volume id:261 size:1076628592 collection:"collection2" file_count:18355 delete_count:1 deleted_byte_count:1155 replica_placement:100 version:3 modified_at_second:1614866420 + volume id:262 size:1078492584 collection:"collection2" file_count:20390 delete_count:3 deleted_byte_count:287601 replica_placement:100 version:3 modified_at_second:1614871601 + volume id:264 size:1081624192 collection:"collection2" file_count:21151 replica_placement:100 version:3 modified_at_second:1614871629 + volume id:265 size:1076401104 collection:"collection2" file_count:19932 delete_count:2 deleted_byte_count:160823 replica_placement:100 version:3 modified_at_second:1614871543 + volume id:266 size:1075617552 collection:"collection2" file_count:20075 delete_count:1 deleted_byte_count:1039 replica_placement:100 version:3 modified_at_second:1614871526 + volume id:267 size:1075699376 collection:"collection2" file_count:21039 delete_count:3 deleted_byte_count:59956 replica_placement:100 version:3 modified_at_second:1614877294 + volume id:270 size:1076876424 collection:"collection2" file_count:22057 delete_count:1 deleted_byte_count:43916 replica_placement:100 version:3 modified_at_second:1614877469 + volume id:271 size:1076992704 collection:"collection2" file_count:22640 delete_count:1 deleted_byte_count:30645 replica_placement:100 version:3 modified_at_second:1614877504 + volume id:272 size:1076145912 collection:"collection2" file_count:21034 delete_count:2 deleted_byte_count:216564 replica_placement:100 version:3 modified_at_second:1614884139 + volume id:273 size:1074873432 collection:"collection2" file_count:20511 delete_count:3 deleted_byte_count:46076 replica_placement:100 version:3 modified_at_second:1614884046 + volume id:274 size:1075994184 collection:"collection2" file_count:20997 replica_placement:100 version:3 modified_at_second:1614884113 + volume id:275 size:1078349024 collection:"collection2" file_count:20808 delete_count:1 deleted_byte_count:1118 replica_placement:100 version:3 modified_at_second:1614884147 + volume id:276 size:1076899880 collection:"collection2" file_count:20190 delete_count:1 deleted_byte_count:8798 replica_placement:100 version:3 modified_at_second:1614884003 + volume id:278 size:1078798632 collection:"collection2" file_count:20597 delete_count:5 deleted_byte_count:400060 replica_placement:100 version:3 modified_at_second:1614890292 + volume id:280 size:1077432160 collection:"collection2" file_count:20286 delete_count:1 deleted_byte_count:879 replica_placement:100 version:3 modified_at_second:1614890262 + volume id:281 size:1077581064 collection:"collection2" file_count:20206 delete_count:3 deleted_byte_count:143964 replica_placement:100 version:3 modified_at_second:1614890237 + volume id:282 size:1075232184 collection:"collection2" file_count:22659 delete_count:4 deleted_byte_count:67915 replica_placement:100 version:3 modified_at_second:1614897304 + volume id:283 size:1080178880 collection:"collection2" file_count:19462 delete_count:7 deleted_byte_count:660407 replica_placement:100 version:3 modified_at_second:1614896623 + volume id:286 size:1077464816 collection:"collection2" file_count:23905 delete_count:6 deleted_byte_count:630577 replica_placement:100 version:3 modified_at_second:1614897401 + volume id:287 size:1074590536 collection:"collection2" file_count:28163 delete_count:5 deleted_byte_count:35727 replica_placement:100 version:3 modified_at_second:1614904875 + volume id:288 size:1075406920 collection:"collection2" file_count:27243 delete_count:2 deleted_byte_count:51519 replica_placement:100 version:3 modified_at_second:1614904738 + volume id:289 size:1075284312 collection:"collection2" file_count:29342 delete_count:5 deleted_byte_count:100454 replica_placement:100 version:3 modified_at_second:1614904977 + volume id:290 size:1074723800 collection:"collection2" file_count:28340 delete_count:4 deleted_byte_count:199064 replica_placement:100 version:3 modified_at_second:1614904924 + volume id:292 size:1092010672 collection:"collection2" file_count:26781 delete_count:5 deleted_byte_count:508910 replica_placement:100 version:3 modified_at_second:1614912325 + volume id:295 size:1074702320 collection:"collection2" file_count:24488 delete_count:3 deleted_byte_count:48555 replica_placement:100 version:3 modified_at_second:1614911929 + volume id:296 size:1077824056 collection:"collection2" file_count:26741 delete_count:4 deleted_byte_count:199906 replica_placement:100 version:3 modified_at_second:1614912301 + volume id:297 size:1080229176 collection:"collection2" file_count:23409 delete_count:5 deleted_byte_count:46268 replica_placement:100 version:3 modified_at_second:1614918481 + volume id:298 size:1075410024 collection:"collection2" file_count:23222 delete_count:2 deleted_byte_count:46110 replica_placement:100 version:3 modified_at_second:1614918474 + volume id:302 size:1077559640 collection:"collection2" file_count:23124 delete_count:7 deleted_byte_count:293111 replica_placement:100 version:3 modified_at_second:1614925500 + volume id:304 size:1081038944 collection:"collection2" file_count:24505 delete_count:2 deleted_byte_count:124447 replica_placement:100 version:3 modified_at_second:1614925569 + volume id:305 size:1074185376 collection:"collection2" file_count:22074 delete_count:5 deleted_byte_count:20221 replica_placement:100 version:3 modified_at_second:1614925312 + volume id:306 size:1074763952 collection:"collection2" file_count:22939 replica_placement:100 version:3 modified_at_second:1614925462 + volume id:307 size:1076567912 collection:"collection2" file_count:23377 delete_count:2 deleted_byte_count:25453 replica_placement:100 version:3 modified_at_second:1614931448 + volume id:308 size:1074022336 collection:"collection2" file_count:23086 delete_count:2 deleted_byte_count:2127 replica_placement:100 version:3 modified_at_second:1614931401 + volume id:311 size:1088248344 collection:"collection2" file_count:23553 delete_count:6 deleted_byte_count:191716 replica_placement:100 version:3 modified_at_second:1614931463 + volume id:312 size:1075037528 collection:"collection2" file_count:22524 replica_placement:100 version:3 modified_at_second:1614937831 + volume id:313 size:1074875960 collection:"collection2" file_count:22404 delete_count:4 deleted_byte_count:51728 replica_placement:100 version:3 modified_at_second:1614937755 + volume id:316 size:1077720776 collection:"collection2" file_count:22605 delete_count:1 deleted_byte_count:8503 replica_placement:100 version:3 modified_at_second:1614937838 + volume id:318 size:1075965168 collection:"collection2" file_count:22459 delete_count:2 deleted_byte_count:37778 replica_placement:100 version:3 modified_at_second:1614943862 + volume id:322 size:1078471536 collection:"collection2" file_count:21905 delete_count:3 deleted_byte_count:145002 replica_placement:100 version:3 modified_at_second:1614950572 + volume id:323 size:1074608056 collection:"collection2" file_count:21605 delete_count:4 deleted_byte_count:172090 replica_placement:100 version:3 modified_at_second:1614950526 + volume id:325 size:1080701232 collection:"collection2" file_count:21735 replica_placement:100 version:3 modified_at_second:1614950525 + volume id:326 size:1076059920 collection:"collection2" file_count:22564 delete_count:2 deleted_byte_count:192886 replica_placement:100 version:3 modified_at_second:1614950619 + volume id:327 size:1076121304 collection:"collection2" file_count:22007 delete_count:3 deleted_byte_count:60358 replica_placement:100 version:3 modified_at_second:1614956487 + volume id:328 size:1074767816 collection:"collection2" file_count:21720 delete_count:3 deleted_byte_count:56429 replica_placement:100 version:3 modified_at_second:1614956362 + volume id:329 size:1076691960 collection:"collection2" file_count:22411 delete_count:5 deleted_byte_count:214092 replica_placement:100 version:3 modified_at_second:1614956485 + volume id:330 size:1080825760 collection:"collection2" file_count:22464 delete_count:2 deleted_byte_count:15771 replica_placement:100 version:3 modified_at_second:1614956476 + volume id:331 size:1074957256 collection:"collection2" file_count:21230 delete_count:4 deleted_byte_count:62145 replica_placement:100 version:3 modified_at_second:1614956259 + volume id:332 size:1075569928 collection:"collection2" file_count:22097 delete_count:3 deleted_byte_count:98273 replica_placement:100 version:3 modified_at_second:1614962869 + volume id:333 size:1074270160 collection:"collection2" file_count:21271 delete_count:2 deleted_byte_count:168122 replica_placement:100 version:3 modified_at_second:1614962697 + volume id:334 size:1075607880 collection:"collection2" file_count:22546 delete_count:6 deleted_byte_count:101538 replica_placement:100 version:3 modified_at_second:1614962978 + volume id:335 size:1076235136 collection:"collection2" file_count:22391 delete_count:3 deleted_byte_count:8838 replica_placement:100 version:3 modified_at_second:1614962970 + volume id:337 size:1075646896 collection:"collection2" file_count:21934 delete_count:1 deleted_byte_count:3397 replica_placement:100 version:3 modified_at_second:1614969937 + volume id:339 size:1078402392 collection:"collection2" file_count:22309 replica_placement:100 version:3 modified_at_second:1614969995 + volume id:340 size:1079462152 collection:"collection2" file_count:22319 delete_count:4 deleted_byte_count:93620 replica_placement:100 version:3 modified_at_second:1614969977 + volume id:341 size:1074448360 collection:"collection2" file_count:21590 delete_count:5 deleted_byte_count:160085 replica_placement:100 version:3 modified_at_second:1614969858 + volume id:343 size:1075345072 collection:"collection2" file_count:21095 delete_count:2 deleted_byte_count:20581 replica_placement:100 version:3 modified_at_second:1614977148 + volume id:346 size:1076464112 collection:"collection2" file_count:22320 delete_count:4 deleted_byte_count:798258 replica_placement:100 version:3 modified_at_second:1614977511 + volume id:347 size:1075145248 collection:"collection2" file_count:22178 delete_count:1 deleted_byte_count:79392 replica_placement:100 version:3 modified_at_second:1614984727 + volume id:348 size:1080623544 collection:"collection2" file_count:21667 delete_count:1 deleted_byte_count:2443 replica_placement:100 version:3 modified_at_second:1614984604 + volume id:349 size:1075957672 collection:"collection2" file_count:22395 delete_count:2 deleted_byte_count:61565 replica_placement:100 version:3 modified_at_second:1614984748 + volume id:351 size:1078795120 collection:"collection2" file_count:23660 delete_count:3 deleted_byte_count:102141 replica_placement:100 version:3 modified_at_second:1614984816 + volume id:352 size:1077145936 collection:"collection2" file_count:22066 delete_count:1 deleted_byte_count:1018 replica_placement:100 version:3 modified_at_second:1614992130 + volume id:353 size:1074897496 collection:"collection2" file_count:21266 delete_count:2 deleted_byte_count:3105374 replica_placement:100 version:3 modified_at_second:1614991951 + volume id:354 size:1085214104 collection:"collection2" file_count:23150 delete_count:4 deleted_byte_count:82391 replica_placement:100 version:3 modified_at_second:1614992208 + volume id:357 size:1074276152 collection:"collection2" file_count:23137 delete_count:4 deleted_byte_count:188487 replica_placement:100 version:3 modified_at_second:1614998792 + volume id:359 size:1074211296 collection:"collection2" file_count:22437 delete_count:2 deleted_byte_count:187953 replica_placement:100 version:3 modified_at_second:1614998711 + volume id:360 size:1075532512 collection:"collection2" file_count:22574 delete_count:3 deleted_byte_count:1774776 replica_placement:100 version:3 modified_at_second:1614998770 + volume id:361 size:1075362744 collection:"collection2" file_count:22272 delete_count:1 deleted_byte_count:3497 replica_placement:100 version:3 modified_at_second:1614998668 + volume id:362 size:1074074176 collection:"collection2" file_count:20595 delete_count:1 deleted_byte_count:112145 replica_placement:100 version:3 modified_at_second:1615004407 + volume id:363 size:1078859640 collection:"collection2" file_count:23177 delete_count:4 deleted_byte_count:9601 replica_placement:100 version:3 modified_at_second:1615004823 + volume id:364 size:1081280880 collection:"collection2" file_count:22686 delete_count:1 deleted_byte_count:84375 replica_placement:100 version:3 modified_at_second:1615004813 + volume id:365 size:1075736632 collection:"collection2" file_count:22193 delete_count:5 deleted_byte_count:259033 replica_placement:100 version:3 modified_at_second:1615004776 + volume id:366 size:1075267272 collection:"collection2" file_count:21856 delete_count:5 deleted_byte_count:138363 replica_placement:100 version:3 modified_at_second:1615004703 + volume id:367 size:1076403648 collection:"collection2" file_count:22995 delete_count:2 deleted_byte_count:36955 replica_placement:100 version:3 modified_at_second:1615010985 + volume id:368 size:1074821960 collection:"collection2" file_count:22252 delete_count:4 deleted_byte_count:3291946 replica_placement:100 version:3 modified_at_second:1615010877 + volume id:369 size:1091472040 collection:"collection2" file_count:23709 delete_count:4 deleted_byte_count:400876 replica_placement:100 version:3 modified_at_second:1615011021 + volume id:370 size:1076040544 collection:"collection2" file_count:22092 delete_count:2 deleted_byte_count:115388 replica_placement:100 version:3 modified_at_second:1615010877 + volume id:371 size:1078806216 collection:"collection2" file_count:22685 delete_count:2 deleted_byte_count:68905 replica_placement:100 version:3 modified_at_second:1615010995 + volume id:372 size:1076193344 collection:"collection2" file_count:22774 delete_count:1 deleted_byte_count:3495 replica_placement:100 version:3 modified_at_second:1615016911 + volume id:373 size:1080928088 collection:"collection2" file_count:22617 delete_count:4 deleted_byte_count:91849 replica_placement:100 version:3 modified_at_second:1615016878 + volume id:374 size:1085011176 collection:"collection2" file_count:23054 delete_count:2 deleted_byte_count:89034 replica_placement:100 version:3 modified_at_second:1615016917 + volume id:376 size:1074845832 collection:"collection2" file_count:22908 delete_count:4 deleted_byte_count:432305 replica_placement:100 version:3 modified_at_second:1615016916 + volume id:377 size:957434264 collection:"collection2" file_count:14929 delete_count:1 deleted_byte_count:43099 replica_placement:100 version:3 modified_at_second:1615632323 + volume id:379 size:1014108528 collection:"collection2" file_count:15362 delete_count:6 deleted_byte_count:2481613 replica_placement:100 version:3 modified_at_second:1615632323 + Disk hdd total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660 + DataNode 192.168.1.5:8080 total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660 + Rack DefaultRack total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660 + DataCenter dc5 total size:306912958016 file_count:4201794 deleted_file:15268 deleted_bytes:4779359660 +total size:775256653592 file_count:10478712 deleted_file:33754 deleted_bytes:10839266043 diff --git a/weed/storage/store_vacuum.go b/weed/storage/store_vacuum.go index 0d6e0b0f1..cbd716b32 100644 --- a/weed/storage/store_vacuum.go +++ b/weed/storage/store_vacuum.go @@ -26,6 +26,9 @@ func (s *Store) CompactVolume(vid needle.VolumeId, preallocate int64, compaction return fmt.Errorf("volume id %d is not found during compact", vid) } func (s *Store) CommitCompactVolume(vid needle.VolumeId) (bool, error) { + if s.isStopping { + return false, fmt.Errorf("volume id %d skips compact because volume is stopping", vid) + } if v := s.findVolume(vid); v != nil { return v.IsReadOnly(), v.CommitCompact() } diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go index d686e2b09..56e8beddb 100644 --- a/weed/storage/volume_vacuum.go +++ b/weed/storage/volume_vacuum.go @@ -289,6 +289,9 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI return fmt.Errorf("ReadNeedleBlob %s key %d offset %d size %d failed: %v", oldDatFile.Name(), key, increIdxEntry.offset.ToActualOffset(), increIdxEntry.size, err) } dstDatBackend.Write(needleBytes) + if err := dstDatBackend.Sync(); err != nil { + return fmt.Errorf("cannot sync needle %s: %v", dstDatBackend.File.Name(), err) + } util.Uint32toBytes(idxEntryBytes[8:12], uint32(offset/NeedlePaddingSize)) } else { //deleted needle //fakeDelNeedle 's default Data field is nil @@ -308,6 +311,12 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI newIdxFileName, err) } _, err = idx.Write(idxEntryBytes) + if err != nil { + return fmt.Errorf("cannot write indexfile %s: %v", newIdxFileName, err) + } + if err := idx.Sync(); err != nil { + return fmt.Errorf("cannot sync indexfile %s: %v", newIdxFileName, err) + } } return nil diff --git a/weed/storage/volume_vacuum_test.go b/weed/storage/volume_vacuum_test.go index 0177cb64d..8212d86c7 100644 --- a/weed/storage/volume_vacuum_test.go +++ b/weed/storage/volume_vacuum_test.go @@ -2,7 +2,6 @@ package storage import ( "math/rand" - "os" "testing" "time" @@ -62,11 +61,7 @@ func TestMakeDiff(t *testing.T) { } func TestCompaction(t *testing.T) { - dir, err := os.MkdirTemp("", "example") - if err != nil { - t.Fatalf("temp dir creation: %v", err) - } - defer os.RemoveAll(dir) // clean up + dir := t.TempDir() v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, 0) if err != nil { diff --git a/weed/storage/volume_write_test.go b/weed/storage/volume_write_test.go index 9f661a27f..11fe49358 100644 --- a/weed/storage/volume_write_test.go +++ b/weed/storage/volume_write_test.go @@ -2,7 +2,6 @@ package storage import ( "fmt" - "os" "testing" "time" @@ -12,11 +11,7 @@ import ( ) func TestSearchVolumesWithDeletedNeedles(t *testing.T) { - dir, err := os.MkdirTemp("", "example") - if err != nil { - t.Fatalf("temp dir creation: %v", err) - } - defer os.RemoveAll(dir) // clean up + dir := t.TempDir() v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, 0) if err != nil { diff --git a/weed/topology/store_replicate.go b/weed/topology/store_replicate.go index b0d063ac9..65625b3b7 100644 --- a/weed/topology/store_replicate.go +++ b/weed/topology/store_replicate.go @@ -101,6 +101,7 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt stats.VolumeServerRequestCounter.WithLabelValues(stats.ErrorWriteToReplicas).Inc() err = fmt.Errorf("failed to write to replicas for volume %d: %v", volumeId, err) glog.V(0).Infoln(err) + return false, err } } return diff --git a/weed/util/chunk_cache/chunk_cache_on_disk_test.go b/weed/util/chunk_cache/chunk_cache_on_disk_test.go index 7dccfd43f..1e7738fa2 100644 --- a/weed/util/chunk_cache/chunk_cache_on_disk_test.go +++ b/weed/util/chunk_cache/chunk_cache_on_disk_test.go @@ -4,14 +4,11 @@ import ( "bytes" "fmt" "math/rand" - "os" "testing" ) func TestOnDisk(t *testing.T) { - - tmpDir, _ := os.MkdirTemp("", "c") - defer os.RemoveAll(tmpDir) + tmpDir := t.TempDir() totalDiskSizeInKB := int64(32) diff --git a/weed/util/constants.go b/weed/util/constants.go index d148e1ca6..ac3491de8 100644 --- a/weed/util/constants.go +++ b/weed/util/constants.go @@ -5,7 +5,7 @@ import ( ) var ( - VERSION_NUMBER = fmt.Sprintf("%.02f", 2.88) + VERSION_NUMBER = fmt.Sprintf("%.02f", 2.89) VERSION = sizeLimit + " " + VERSION_NUMBER COMMIT = "" ) diff --git a/weed/util/parse.go b/weed/util/parse.go index 0955db682..502f3a80f 100644 --- a/weed/util/parse.go +++ b/weed/util/parse.go @@ -61,3 +61,8 @@ func ParseHostPort(hostPort string) (filerServer string, filerPort int64, err er return } + +func CanonicalizeETag(etag string) string { + canonicalETag := strings.TrimPrefix(etag, "\"") + return strings.TrimSuffix(canonicalETag, "\"") +} |