From 0983060a9080e69dc7edf58f172ab170755c3511 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 13 Aug 2020 09:07:22 -0700 Subject: increase default volume file size limit to 1024 avoid possible large chunk size set on mount or filer --- weed/command/server.go | 2 +- weed/command/volume.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'weed/command') diff --git a/weed/command/server.go b/weed/command/server.go index 565563c77..d16095075 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -96,7 +96,7 @@ func init() { serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.") serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.") serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second") - serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 256, "limit file size to avoid out of memory") + serverOptions.v.fileSizeLimitMB = cmdServer.Flag.Int("volume.fileSizeLimitMB", 1024, "limit file size to avoid out of memory") serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address") serverOptions.v.pprof = &False diff --git a/weed/command/volume.go b/weed/command/volume.go index 4f04a467d..6fb7447e7 100644 --- a/weed/command/volume.go +++ b/weed/command/volume.go @@ -76,7 +76,7 @@ func init() { v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file") v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file") v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second") - v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 256, "limit file size to avoid out of memory") + v.fileSizeLimitMB = cmdVolume.Flag.Int("fileSizeLimitMB", 1024, "limit file size to avoid out of memory") v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile") } -- cgit v1.2.3 From 6a92f0bc7a2cbf0828c720422220b600263b5217 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 18 Aug 2020 17:04:28 -0700 Subject: refactoring to typed Size Go is amazing with refactoring! --- weed/command/export.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'weed/command') diff --git a/weed/command/export.go b/weed/command/export.go index 411d231cb..0e2e7ccd9 100644 --- a/weed/command/export.go +++ b/weed/command/export.go @@ -72,9 +72,9 @@ var ( func printNeedle(vid needle.VolumeId, n *needle.Needle, version needle.Version, deleted bool) { key := needle.NewFileIdFromNeedle(vid, n).String() - size := n.DataSize + size := int32(n.DataSize) if version == needle.Version1 { - size = n.Size + size = int32(n.Size) } fmt.Printf("%s\t%s\t%d\t%t\t%s\t%s\t%s\t%t\n", key, -- cgit v1.2.3 From fe01191b5b93c7f3851e7add9b046c89c8189cdc Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 18 Aug 2020 19:22:16 -0700 Subject: support read option readDeleted=true --- weed/command/export.go | 2 +- weed/command/fix.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'weed/command') diff --git a/weed/command/export.go b/weed/command/export.go index 0e2e7ccd9..3ea4b00d3 100644 --- a/weed/command/export.go +++ b/weed/command/export.go @@ -111,7 +111,7 @@ func (scanner *VolumeFileScanner4Export) VisitNeedle(n *needle.Needle, offset in nv, ok := needleMap.Get(n.Id) glog.V(3).Infof("key %d offset %d size %d disk_size %d compressed %v ok %v nv %+v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsCompressed(), ok, nv) - if ok && nv.Size > 0 && nv.Size != types.TombstoneFileSize && nv.Offset.ToAcutalOffset() == offset { + if ok && nv.Size.IsValid() && nv.Offset.ToAcutalOffset() == offset { if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) { glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d", n.LastModified, newerThanUnix) diff --git a/weed/command/fix.go b/weed/command/fix.go index e1455790f..ae9a051b8 100644 --- a/weed/command/fix.go +++ b/weed/command/fix.go @@ -48,7 +48,7 @@ func (scanner *VolumeFileScanner4Fix) ReadNeedleBody() bool { func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error { glog.V(2).Infof("key %d offset %d size %d disk_size %d compressed %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsCompressed()) - if n.Size > 0 && n.Size != types.TombstoneFileSize { + if n.Size.IsValid() { pe := scanner.nm.Set(n.Id, types.ToOffset(offset), n.Size) glog.V(2).Infof("saved %d with error %v", n.Size, pe) } else { -- cgit v1.2.3