Merge pull request #1 from chrislusf/master

sync

1 files changed, 86 insertions, 16 deletions

diff --git a/weed/s3api/filer_util.go b/weed/s3api/filer_util.go
index b93b603e2..ec1eedcb4 100644
--- a/weed/s3api/filer_util.go
+++ b/weed/s3api/filer_util.go
@@ -3,6 +3,7 @@ package s3api
 import (
 	"context"
 	"fmt"
+	"io"
 	"os"
 	"strings"
 	"time"
@@ -11,8 +12,8 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 )
 
-func (s3a *S3ApiServer) mkdir(ctx context.Context, parentDirectoryPath string, dirName string, fn func(entry *filer_pb.Entry)) error {
-	return s3a.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+func (s3a *S3ApiServer) mkdir(parentDirectoryPath string, dirName string, fn func(entry *filer_pb.Entry)) error {
+	return s3a.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
 		entry := &filer_pb.Entry{
 			Name:        dirName,
@@ -36,7 +37,7 @@ func (s3a *S3ApiServer) mkdir(ctx context.Context, parentDirectoryPath string, d
 		}
 
 		glog.V(1).Infof("mkdir: %v", request)
-		if _, err := client.CreateEntry(ctx, request); err != nil {
+		if err := filer_pb.CreateEntry(client, request); err != nil {
 			glog.V(0).Infof("mkdir %v: %v", request, err)
 			return fmt.Errorf("mkdir %s/%s: %v", parentDirectoryPath, dirName, err)
 		}
@@ -45,8 +46,8 @@ func (s3a *S3ApiServer) mkdir(ctx context.Context, parentDirectoryPath string, d
 	})
 }
 
-func (s3a *S3ApiServer) mkFile(ctx context.Context, parentDirectoryPath string, fileName string, chunks []*filer_pb.FileChunk) error {
-	return s3a.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+func (s3a *S3ApiServer) mkFile(parentDirectoryPath string, fileName string, chunks []*filer_pb.FileChunk) error {
+	return s3a.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
 		entry := &filer_pb.Entry{
 			Name:        fileName,
@@ -67,7 +68,7 @@ func (s3a *S3ApiServer) mkFile(ctx context.Context, parentDirectoryPath string,
 		}
 
 		glog.V(1).Infof("create file: %s/%s", parentDirectoryPath, fileName)
-		if _, err := client.CreateEntry(ctx, request); err != nil {
+		if err := filer_pb.CreateEntry(client, request); err != nil {
 			glog.V(0).Infof("create file %v:%v", request, err)
 			return fmt.Errorf("create file %s/%s: %v", parentDirectoryPath, fileName, err)
 		}
@@ -76,9 +77,9 @@ func (s3a *S3ApiServer) mkFile(ctx context.Context, parentDirectoryPath string,
 	})
 }
 
-func (s3a *S3ApiServer) list(ctx context.Context, parentDirectoryPath, prefix, startFrom string, inclusive bool, limit int) (entries []*filer_pb.Entry, err error) {
+func (s3a *S3ApiServer) list(parentDirectoryPath, prefix, startFrom string, inclusive bool, limit int) (entries []*filer_pb.Entry, err error) {
 
-	err = s3a.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+	err = s3a.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.ListEntriesRequest{
 			Directory:          parentDirectoryPath,
@@ -89,13 +90,25 @@ func (s3a *S3ApiServer) list(ctx context.Context, parentDirectoryPath, prefix, s
 		}
 
 		glog.V(4).Infof("read directory: %v", request)
-		resp, err := client.ListEntries(ctx, request)
+		stream, err := client.ListEntries(context.Background(), request)
 		if err != nil {
 			glog.V(0).Infof("read directory %v: %v", request, err)
 			return fmt.Errorf("list dir %v: %v", parentDirectoryPath, err)
 		}
 
-		entries = resp.Entries
+		for {
+			resp, recvErr := stream.Recv()
+			if recvErr != nil {
+				if recvErr == io.EOF {
+					break
+				} else {
+					return recvErr
+				}
+			}
+
+			entries = append(entries, resp.Entry)
+
+		}
 
 		return nil
 	})
@@ -104,9 +117,9 @@ func (s3a *S3ApiServer) list(ctx context.Context, parentDirectoryPath, prefix, s
 
 }
 
-func (s3a *S3ApiServer) rm(ctx context.Context, parentDirectoryPath string, entryName string, isDirectory, isDeleteData, isRecursive bool) error {
+func (s3a *S3ApiServer) rm(parentDirectoryPath, entryName string, isDirectory, isDeleteData, isRecursive bool) error {
 
-	return s3a.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+	return s3a.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.DeleteEntryRequest{
 			Directory:    parentDirectoryPath,
@@ -116,7 +129,7 @@ func (s3a *S3ApiServer) rm(ctx context.Context, parentDirectoryPath string, entr
 		}
 
 		glog.V(1).Infof("delete entry %v/%v: %v", parentDirectoryPath, entryName, request)
-		if _, err := client.DeleteEntry(ctx, request); err != nil {
+		if _, err := client.DeleteEntry(context.Background(), request); err != nil {
 			glog.V(0).Infof("delete entry %v: %v", request, err)
 			return fmt.Errorf("delete entry %s/%s: %v", parentDirectoryPath, entryName, err)
 		}
@@ -126,9 +139,62 @@ func (s3a *S3ApiServer) rm(ctx context.Context, parentDirectoryPath string, entr
 
 }
 
-func (s3a *S3ApiServer) exists(ctx context.Context, parentDirectoryPath string, entryName string, isDirectory bool) (exists bool, err error) {
+func (s3a *S3ApiServer) streamRemove(quiet bool, fn func() (finished bool, parentDirectoryPath string, entryName string, isDeleteData, isRecursive bool), respFn func(err string)) error {
+
+	return s3a.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+
+		stream, err := client.StreamDeleteEntries(context.Background())
+		if err != nil {
+			glog.V(0).Infof("stream delete entry: %v", err)
+			return fmt.Errorf("stream delete entry: %v", err)
+		}
+
+		waitc := make(chan struct{})
+		go func() {
+			for {
+				resp, err := stream.Recv()
+				if err == io.EOF {
+					// read done.
+					close(waitc)
+					return
+				}
+				if err != nil {
+					glog.V(0).Infof("streamRemove: %v", err)
+					return
+				}
+				respFn(resp.Error)
+			}
+		}()
+
+		for {
+			finished, parentDirectoryPath, entryName, isDeleteData, isRecursive := fn()
+			if finished {
+				break
+			}
+			err = stream.Send(&filer_pb.DeleteEntryRequest{
+				Directory:            parentDirectoryPath,
+				Name:                 entryName,
+				IsDeleteData:         isDeleteData,
+				IsRecursive:          isRecursive,
+				IgnoreRecursiveError: quiet,
+			})
+			if err != nil {
+				glog.V(0).Infof("streamRemove: %v", err)
+				break
+			}
+
+		}
+		stream.CloseSend()
+		<-waitc
+		return err
+
+	})
+
+}
+
+func (s3a *S3ApiServer) exists(parentDirectoryPath string, entryName string, isDirectory bool) (exists bool, err error) {
 
-	err = s3a.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+	err = s3a.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.LookupDirectoryEntryRequest{
 			Directory: parentDirectoryPath,
@@ -136,8 +202,12 @@ func (s3a *S3ApiServer) exists(ctx context.Context, parentDirectoryPath string,
 		}
 
 		glog.V(4).Infof("exists entry %v/%v: %v", parentDirectoryPath, entryName, request)
-		resp, err := client.LookupDirectoryEntry(ctx, request)
+		resp, err := filer_pb.LookupEntry(client, request)
 		if err != nil {
+			if err == filer_pb.ErrNotFound {
+				exists = false
+				return nil
+			}
 			glog.V(0).Infof("exists entry %v: %v", request, err)
 			return fmt.Errorf("exists entry %s/%s: %v", parentDirectoryPath, entryName, err)
 		}