filer read empty file may cause OOM in some cases

fix https://github.com/chrislusf/seaweedfs/issues/2641

5 files changed, 10 insertions, 16 deletions

diff --git a/weed/command/filer_cat.go b/weed/command/filer_cat.go
index 7f613f72b..ada843dea 100644
--- a/weed/command/filer_cat.go
+++ b/weed/command/filer_cat.go
@@ -8,7 +8,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
 	"google.golang.org/grpc"
-	"math"
 	"net/url"
 	"os"
 	"strings"
@@ -115,7 +114,7 @@ func runFilerCat(cmd *Command, args []string) bool {
 
 		filerCat.filerClient = client
 
-		return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
+		return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, int64(filer.FileSize(respLookupEntry.Entry)))
 
 	})
 
diff --git a/weed/filer/filer_conf.go b/weed/filer/filer_conf.go
index 45c368b9b..32fc647d9 100644
--- a/weed/filer/filer_conf.go
+++ b/weed/filer/filer_conf.go
@@ -75,12 +75,12 @@ func (fc *FilerConf) loadFromFiler(filer *Filer) (err error) {
 		return fc.LoadFromBytes(entry.Content)
 	}
 
-	return fc.loadFromChunks(filer, entry.Content, entry.Chunks)
+	return fc.loadFromChunks(filer, entry.Content, entry.Chunks, entry.Size())
 }
 
-func (fc *FilerConf) loadFromChunks(filer *Filer, content []byte, chunks []*filer_pb.FileChunk) (err error) {
+func (fc *FilerConf) loadFromChunks(filer *Filer, content []byte, chunks []*filer_pb.FileChunk, size uint64) (err error) {
 	if len(content) == 0 {
-		content, err = filer.readEntry(chunks)
+		content, err = filer.readEntry(chunks, size)
 		if err != nil {
 			glog.Errorf("read filer conf content: %v", err)
 			return
diff --git a/weed/filer/filer_on_meta_event.go b/weed/filer/filer_on_meta_event.go
index 34ac5321a..720e019f4 100644
--- a/weed/filer/filer_on_meta_event.go
+++ b/weed/filer/filer_on_meta_event.go
@@ -2,8 +2,6 @@ package filer
 
 import (
 	"bytes"
-	"math"
-
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
@@ -55,9 +53,9 @@ func (f *Filer) maybeReloadFilerConfiguration(event *filer_pb.SubscribeMetadataR
 	}
 }
 
-func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) {
+func (f *Filer) readEntry(chunks []*filer_pb.FileChunk, size uint64) ([]byte, error) {
 	var buf bytes.Buffer
-	err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64)
+	err := StreamContent(f.MasterClient, &buf, chunks, 0, int64(size))
 	if err != nil {
 		return nil, err
 	}
@@ -66,7 +64,7 @@ func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) {
 
 func (f *Filer) reloadFilerConfiguration(entry *filer_pb.Entry) {
 	fc := NewFilerConf()
-	err := fc.loadFromChunks(f, entry.Content, entry.Chunks)
+	err := fc.loadFromChunks(f, entry.Content, entry.Chunks, FileSize(entry))
 	if err != nil {
 		glog.Errorf("read filer conf chunks: %v", err)
 		return
diff --git a/weed/filer/read_write.go b/weed/filer/read_write.go
index 14e8cab1e..3b6a69fb6 100644
--- a/weed/filer/read_write.go
+++ b/weed/filer/read_write.go
@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
-	"math"
 	"time"
 )
 
@@ -23,7 +22,7 @@ func ReadEntry(masterClient *wdclient.MasterClient, filerClient filer_pb.Seaweed
 		return err
 	}
 
-	return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
+	return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, int64(FileSize(respLookupEntry.Entry)))
 
 }
 
diff --git a/weed/shell/command_fs_cat.go b/weed/shell/command_fs_cat.go
index 17e9c6550..16be25ee5 100644
--- a/weed/shell/command_fs_cat.go
+++ b/weed/shell/command_fs_cat.go
@@ -2,12 +2,10 @@ package shell
 
 import (
 	"fmt"
-	"io"
-	"math"
-
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"io"
 )
 
 func init() {
@@ -57,7 +55,7 @@ func (c *commandFsCat) Do(args []string, commandEnv *CommandEnv, writer io.Write
 			return err
 		}
 
-		return filer.StreamContent(commandEnv.MasterClient, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
+		return filer.StreamContent(commandEnv.MasterClient, writer, respLookupEntry.Entry.Chunks, 0, int64(filer.FileSize(respLookupEntry.Entry)))
 
 	})