async file chunk deletion

1 files changed, 54 insertions, 7 deletions

diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go
index 8e1d6fadd..8909da52a 100644
--- a/weed/filesys/filehandle.go
+++ b/weed/filesys/filehandle.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/seaweedfs/fuse"
@@ -159,7 +160,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f
 	for _, chunk := range chunks {
 		fh.f.entry.Chunks = append(fh.f.entry.Chunks, chunk)
 		fh.f.entryViewCache = nil
-		glog.V(1).Infof("uploaded %s/%s to %s [%d,%d)", fh.f.dir.Path, fh.f.Name, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size))
+		glog.V(4).Infof("uploaded %s/%s to %s [%d,%d)", fh.f.dir.Path, fh.f.Name, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size))
 		fh.dirtyMetadata = true
 	}
 
@@ -179,8 +180,6 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err
 	return nil
 }
 
-// Flush - experimenting with uploading at flush, this slows operations down till it has been
-// completely flushed
 func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 	// fflush works at fh level
 	// send the data to the OS
@@ -216,10 +215,16 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 			Entry:     fh.f.entry,
 		}
 
-		glog.V(1).Infof("%s/%s set chunks: %v", fh.f.dir.Path, fh.f.Name, len(fh.f.entry.Chunks))
-		for i, chunk := range fh.f.entry.Chunks {
-			glog.V(1).Infof("%s/%s chunks %d: %v [%d,%d)", fh.f.dir.Path, fh.f.Name, i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size))
-		}
+		//glog.V(1).Infof("%s/%s set chunks: %v", fh.f.dir.Path, fh.f.Name, len(fh.f.entry.Chunks))
+		//for i, chunk := range fh.f.entry.Chunks {
+		//	glog.V(4).Infof("%s/%s chunks %d: %v [%d,%d)", fh.f.dir.Path, fh.f.Name, i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size))
+		//}
+
+		chunks, garbages := filer2.CompactFileChunks(fh.f.entry.Chunks)
+		fh.f.entry.Chunks = chunks
+		fh.f.entryViewCache = nil
+		fh.f.wfs.asyncDeleteFileChunks(garbages)
+
 		if _, err := client.CreateEntry(ctx, request); err != nil {
 			return fmt.Errorf("update fh: %v", err)
 		}
@@ -228,6 +233,48 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 	})
 }
 
+func deleteFileIds(ctx context.Context, client filer_pb.SeaweedFilerClient, fileIds []string) error {
+
+	var vids []string
+	for _, fileId := range fileIds {
+		vids = append(vids, volumeId(fileId))
+	}
+
+	lookupFunc := func(vids []string) (map[string]operation.LookupResult, error) {
+
+		m := make(map[string]operation.LookupResult)
+
+		glog.V(4).Infof("remove file lookup volume id locations: %v", vids)
+		resp, err := client.LookupVolume(ctx, &filer_pb.LookupVolumeRequest{
+			VolumeIds: vids,
+		})
+		if err != nil {
+			return m, err
+		}
+
+		for _, vid := range vids {
+			lr := operation.LookupResult{
+				VolumeId:  vid,
+				Locations: nil,
+			}
+			locations := resp.LocationsMap[vid]
+			for _, loc := range locations.Locations {
+				lr.Locations = append(lr.Locations, operation.Location{
+					Url:       loc.Url,
+					PublicUrl: loc.PublicUrl,
+				})
+			}
+			m[vid] = lr
+		}
+
+		return m, err
+	}
+
+	_, err := operation.DeleteFilesWithLookupVolumeId(fileIds, lookupFunc)
+
+	return err
+}
+
 func volumeId(fileId string) string {
 	lastCommaIndex := strings.LastIndex(fileId, ",")
 	if lastCommaIndex > 0 {