1 files changed, 91 insertions, 12 deletions

diff --git a/weed/s3api/filer_util.go b/weed/s3api/filer_util.go
index 6dd93d50f..ec1eedcb4 100644
--- a/weed/s3api/filer_util.go
+++ b/weed/s3api/filer_util.go
@@ -3,7 +3,9 @@ package s3api
 import (
 	"context"
 	"fmt"
+	"io"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -35,7 +37,8 @@ func (s3a *S3ApiServer) mkdir(parentDirectoryPath string, dirName string, fn fun
 		}
 
 		glog.V(1).Infof("mkdir: %v", request)
-		if _, err := client.CreateEntry(context.Background(), request); err != nil {
+		if err := filer_pb.CreateEntry(client, request); err != nil {
+			glog.V(0).Infof("mkdir %v: %v", request, err)
 			return fmt.Errorf("mkdir %s/%s: %v", parentDirectoryPath, dirName, err)
 		}
 
@@ -65,7 +68,8 @@ func (s3a *S3ApiServer) mkFile(parentDirectoryPath string, fileName string, chun
 		}
 
 		glog.V(1).Infof("create file: %s/%s", parentDirectoryPath, fileName)
-		if _, err := client.CreateEntry(context.Background(), request); err != nil {
+		if err := filer_pb.CreateEntry(client, request); err != nil {
+			glog.V(0).Infof("create file %v:%v", request, err)
 			return fmt.Errorf("create file %s/%s: %v", parentDirectoryPath, fileName, err)
 		}
 
@@ -86,12 +90,25 @@ func (s3a *S3ApiServer) list(parentDirectoryPath, prefix, startFrom string, incl
 		}
 
 		glog.V(4).Infof("read directory: %v", request)
-		resp, err := client.ListEntries(context.Background(), request)
+		stream, err := client.ListEntries(context.Background(), request)
 		if err != nil {
+			glog.V(0).Infof("read directory %v: %v", request, err)
 			return fmt.Errorf("list dir %v: %v", parentDirectoryPath, err)
 		}
 
-		entries = resp.Entries
+		for {
+			resp, recvErr := stream.Recv()
+			if recvErr != nil {
+				if recvErr == io.EOF {
+					break
+				} else {
+					return recvErr
+				}
+			}
+
+			entries = append(entries, resp.Entry)
+
+		}
 
 		return nil
 	})
@@ -100,22 +117,20 @@ func (s3a *S3ApiServer) list(parentDirectoryPath, prefix, startFrom string, incl
 
 }
 
-func (s3a *S3ApiServer) rm(parentDirectoryPath string, entryName string, isDirectory, isDeleteData, isRecursive bool) error {
+func (s3a *S3ApiServer) rm(parentDirectoryPath, entryName string, isDirectory, isDeleteData, isRecursive bool) error {
 
 	return s3a.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
-		ctx := context.Background()
-
 		request := &filer_pb.DeleteEntryRequest{
 			Directory:    parentDirectoryPath,
 			Name:         entryName,
-			IsDirectory:  isDirectory,
 			IsDeleteData: isDeleteData,
 			IsRecursive:  isRecursive,
 		}
 
 		glog.V(1).Infof("delete entry %v/%v: %v", parentDirectoryPath, entryName, request)
-		if _, err := client.DeleteEntry(ctx, request); err != nil {
+		if _, err := client.DeleteEntry(context.Background(), request); err != nil {
+			glog.V(0).Infof("delete entry %v: %v", request, err)
 			return fmt.Errorf("delete entry %s/%s: %v", parentDirectoryPath, entryName, err)
 		}
 
@@ -124,20 +139,76 @@ func (s3a *S3ApiServer) rm(parentDirectoryPath string, entryName string, isDirec
 
 }
 
+func (s3a *S3ApiServer) streamRemove(quiet bool, fn func() (finished bool, parentDirectoryPath string, entryName string, isDeleteData, isRecursive bool), respFn func(err string)) error {
+
+	return s3a.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+
+		stream, err := client.StreamDeleteEntries(context.Background())
+		if err != nil {
+			glog.V(0).Infof("stream delete entry: %v", err)
+			return fmt.Errorf("stream delete entry: %v", err)
+		}
+
+		waitc := make(chan struct{})
+		go func() {
+			for {
+				resp, err := stream.Recv()
+				if err == io.EOF {
+					// read done.
+					close(waitc)
+					return
+				}
+				if err != nil {
+					glog.V(0).Infof("streamRemove: %v", err)
+					return
+				}
+				respFn(resp.Error)
+			}
+		}()
+
+		for {
+			finished, parentDirectoryPath, entryName, isDeleteData, isRecursive := fn()
+			if finished {
+				break
+			}
+			err = stream.Send(&filer_pb.DeleteEntryRequest{
+				Directory:            parentDirectoryPath,
+				Name:                 entryName,
+				IsDeleteData:         isDeleteData,
+				IsRecursive:          isRecursive,
+				IgnoreRecursiveError: quiet,
+			})
+			if err != nil {
+				glog.V(0).Infof("streamRemove: %v", err)
+				break
+			}
+
+		}
+		stream.CloseSend()
+		<-waitc
+		return err
+
+	})
+
+}
+
 func (s3a *S3ApiServer) exists(parentDirectoryPath string, entryName string, isDirectory bool) (exists bool, err error) {
 
 	err = s3a.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
-		ctx := context.Background()
-
 		request := &filer_pb.LookupDirectoryEntryRequest{
 			Directory: parentDirectoryPath,
 			Name:      entryName,
 		}
 
 		glog.V(4).Infof("exists entry %v/%v: %v", parentDirectoryPath, entryName, request)
-		resp, err := client.LookupDirectoryEntry(ctx, request)
+		resp, err := filer_pb.LookupEntry(client, request)
 		if err != nil {
+			if err == filer_pb.ErrNotFound {
+				exists = false
+				return nil
+			}
+			glog.V(0).Infof("exists entry %v: %v", request, err)
 			return fmt.Errorf("exists entry %s/%s: %v", parentDirectoryPath, entryName, err)
 		}
 
@@ -148,3 +219,11 @@ func (s3a *S3ApiServer) exists(parentDirectoryPath string, entryName string, isD
 
 	return
 }
+
+func objectKey(key *string) *string {
+	if strings.HasPrefix(*key, "/") {
+		t := (*key)[1:]
+		return &t
+	}
+	return key
+}