aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--weed/command/backup.go27
-rw-r--r--weed/command/export.go2
-rw-r--r--weed/command/fix.go2
-rw-r--r--weed/operation/sync_volume.go41
-rw-r--r--weed/pb/volume_server.proto20
-rw-r--r--weed/pb/volume_server_pb/volume_server.pb.go380
-rw-r--r--weed/server/volume_grpc_follow.go67
-rw-r--r--weed/server/volume_grpc_replicate.go2
-rw-r--r--weed/server/volume_grpc_sync.go101
-rw-r--r--weed/shell/command_fs_du.go6
-rw-r--r--weed/storage/store_vacuum.go2
-rw-r--r--weed/storage/volume_follow.go253
-rw-r--r--weed/storage/volume_follow_test.go39
-rw-r--r--weed/storage/volume_read_write.go28
-rw-r--r--weed/storage/volume_sync.go225
-rw-r--r--weed/storage/volume_vacuum.go6
-rw-r--r--weed/storage/volume_vacuum_test.go2
17 files changed, 545 insertions, 658 deletions
diff --git a/weed/command/backup.go b/weed/command/backup.go
index 86391f9c4..48e2eba89 100644
--- a/weed/command/backup.go
+++ b/weed/command/backup.go
@@ -88,7 +88,32 @@ func runBackup(cmd *Command, args []string) bool {
return true
}
- if err := v.Synchronize(volumeServer, grpcDialOption); err != nil {
+ if v.SuperBlock.CompactRevision < uint16(stats.CompactRevision) {
+ if err = v.Compact(0); err != nil {
+ fmt.Printf("Compact Volume before synchronizing %v\n", err)
+ return true
+ }
+ if err = v.CommitCompact(); err != nil {
+ fmt.Printf("Commit Compact before synchronizing %v\n", err)
+ return true
+ }
+ v.SuperBlock.CompactRevision = uint16(stats.CompactRevision)
+ v.DataFile().WriteAt(v.SuperBlock.Bytes(), 0)
+ }
+
+ if uint64(v.Size()) > stats.TailOffset {
+ // remove the old data
+ v.Destroy()
+ // recreate an empty volume
+ v, err = storage.NewVolume(*s.dir, *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0)
+ if err != nil {
+ fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
+ return true
+ }
+ }
+ defer v.Close()
+
+ if err := v.Follow(volumeServer, grpcDialOption); err != nil {
fmt.Printf("Error synchronizing volume %d: %v\n", vid, err)
return true
}
diff --git a/weed/command/export.go b/weed/command/export.go
index 5c7e064ce..cdced5936 100644
--- a/weed/command/export.go
+++ b/weed/command/export.go
@@ -107,7 +107,7 @@ func (scanner *VolumeFileScanner4Export) VisitNeedle(n *storage.Needle, offset i
nv, ok := needleMap.Get(n.Id)
glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsGzipped(), ok, nv)
- if ok && nv.Size > 0 && int64(nv.Offset)*types.NeedlePaddingSize == offset {
+ if ok && nv.Size > 0 && nv.Size != types.TombstoneFileSize && int64(nv.Offset)*types.NeedlePaddingSize == offset {
if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) {
glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
n.LastModified, newerThanUnix)
diff --git a/weed/command/fix.go b/weed/command/fix.go
index a800978c6..42ae23a3c 100644
--- a/weed/command/fix.go
+++ b/weed/command/fix.go
@@ -44,7 +44,7 @@ func (scanner *VolumeFileScanner4Fix) ReadNeedleBody() bool {
func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *storage.Needle, offset int64) error {
glog.V(2).Infof("key %d offset %d size %d disk_size %d gzip %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsGzipped())
- if n.Size > 0 {
+ if n.Size > 0 && n.Size != types.TombstoneFileSize {
pe := scanner.nm.Put(n.Id, types.Offset(offset/types.NeedlePaddingSize), n.Size)
glog.V(2).Infof("saved %d with error %v", n.Size, pe)
} else {
diff --git a/weed/operation/sync_volume.go b/weed/operation/sync_volume.go
index 6af2404c0..5562f12ab 100644
--- a/weed/operation/sync_volume.go
+++ b/weed/operation/sync_volume.go
@@ -2,12 +2,8 @@ package operation
import (
"context"
- "fmt"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
- . "github.com/chrislusf/seaweedfs/weed/storage/types"
- "github.com/chrislusf/seaweedfs/weed/util"
"google.golang.org/grpc"
- "io"
)
func GetVolumeSyncStatus(server string, grpcDialOption grpc.DialOption, vid uint32) (resp *volume_server_pb.VolumeSyncStatusResponse, err error) {
@@ -22,40 +18,3 @@ func GetVolumeSyncStatus(server string, grpcDialOption grpc.DialOption, vid uint
return
}
-
-func GetVolumeIdxEntries(server string, grpcDialOption grpc.DialOption, vid uint32, eachEntryFn func(key NeedleId, offset Offset, size uint32)) error {
-
- return WithVolumeServerClient(server, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
- stream, err := client.VolumeSyncIndex(context.Background(), &volume_server_pb.VolumeSyncIndexRequest{
- VolumeId: vid,
- })
- if err != nil {
- return err
- }
-
- var indexFileContent []byte
-
- for {
- resp, err := stream.Recv()
- if err == io.EOF {
- break
- }
- if err != nil {
- return fmt.Errorf("read index entries: %v", err)
- }
- indexFileContent = append(indexFileContent, resp.IndexFileContent...)
- }
-
- dataSize := len(indexFileContent)
-
- for idx := 0; idx+NeedleEntrySize <= dataSize; idx += NeedleEntrySize {
- line := indexFileContent[idx : idx+NeedleEntrySize]
- key := BytesToNeedleId(line[:NeedleIdSize])
- offset := BytesToOffset(line[NeedleIdSize : NeedleIdSize+OffsetSize])
- size := util.BytesToUint32(line[NeedleIdSize+OffsetSize : NeedleIdSize+OffsetSize+SizeSize])
- eachEntryFn(key, offset, size)
- }
-
- return nil
- })
-}
diff --git a/weed/pb/volume_server.proto b/weed/pb/volume_server.proto
index 93db5b981..3b5b36a21 100644
--- a/weed/pb/volume_server.proto
+++ b/weed/pb/volume_server.proto
@@ -24,9 +24,7 @@ service VolumeServer {
rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) {
}
- rpc VolumeSyncIndex (VolumeSyncIndexRequest) returns (stream VolumeSyncIndexResponse) {
- }
- rpc VolumeSyncData (VolumeSyncDataRequest) returns (stream VolumeSyncDataResponse) {
+ rpc VolumeFollow (VolumeFollowRequest) returns (stream VolumeFollowResponse) {
}
rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) {
@@ -119,21 +117,11 @@ message VolumeSyncStatusResponse {
uint64 idx_file_size = 8;
}
-message VolumeSyncIndexRequest {
- uint32 volume_id = 1;
-}
-message VolumeSyncIndexResponse {
- bytes index_file_content = 1;
-}
-
-message VolumeSyncDataRequest {
+message VolumeFollowRequest {
uint32 volume_id = 1;
- uint32 revision = 2;
- uint32 offset = 3;
- uint32 size = 4;
- string needle_id = 5;
+ uint64 since = 2;
}
-message VolumeSyncDataResponse {
+message VolumeFollowResponse {
bytes file_content = 1;
}
diff --git a/weed/pb/volume_server_pb/volume_server.pb.go b/weed/pb/volume_server_pb/volume_server.pb.go
index d84a5b099..0f3b47ee0 100644
--- a/weed/pb/volume_server_pb/volume_server.pb.go
+++ b/weed/pb/volume_server_pb/volume_server.pb.go
@@ -27,10 +27,8 @@ It has these top-level messages:
AssignVolumeResponse
VolumeSyncStatusRequest
VolumeSyncStatusResponse
- VolumeSyncIndexRequest
- VolumeSyncIndexResponse
- VolumeSyncDataRequest
- VolumeSyncDataResponse
+ VolumeFollowRequest
+ VolumeFollowResponse
VolumeMountRequest
VolumeMountResponse
VolumeUnmountRequest
@@ -420,96 +418,40 @@ func (m *VolumeSyncStatusResponse) GetIdxFileSize() uint64 {
return 0
}
-type VolumeSyncIndexRequest struct {
+type VolumeFollowRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
+ Since uint64 `protobuf:"varint,2,opt,name=since" json:"since,omitempty"`
}
-func (m *VolumeSyncIndexRequest) Reset() { *m = VolumeSyncIndexRequest{} }
-func (m *VolumeSyncIndexRequest) String() string { return proto.CompactTextString(m) }
-func (*VolumeSyncIndexRequest) ProtoMessage() {}
-func (*VolumeSyncIndexRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
+func (m *VolumeFollowRequest) Reset() { *m = VolumeFollowRequest{} }
+func (m *VolumeFollowRequest) String() string { return proto.CompactTextString(m) }
+func (*VolumeFollowRequest) ProtoMessage() {}
+func (*VolumeFollowRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
-func (m *VolumeSyncIndexRequest) GetVolumeId() uint32 {
+func (m *VolumeFollowRequest) GetVolumeId() uint32 {
if m != nil {
return m.VolumeId
}
return 0
}
-type VolumeSyncIndexResponse struct {
- IndexFileContent []byte `protobuf:"bytes,1,opt,name=index_file_content,json=indexFileContent,proto3" json:"index_file_content,omitempty"`
-}
-
-func (m *VolumeSyncIndexResponse) Reset() { *m = VolumeSyncIndexResponse{} }
-func (m *VolumeSyncIndexResponse) String() string { return proto.CompactTextString(m) }
-func (*VolumeSyncIndexResponse) ProtoMessage() {}
-func (*VolumeSyncIndexResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
-
-func (m *VolumeSyncIndexResponse) GetIndexFileContent() []byte {
- if m != nil {
- return m.IndexFileContent
- }
- return nil
-}
-
-type VolumeSyncDataRequest struct {
- VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
- Revision uint32 `protobuf:"varint,2,opt,name=revision" json:"revision,omitempty"`
- Offset uint32 `protobuf:"varint,3,opt,name=offset" json:"offset,omitempty"`
- Size uint32 `protobuf:"varint,4,opt,name=size" json:"size,omitempty"`
- NeedleId string `protobuf:"bytes,5,opt,name=needle_id,json=needleId" json:"needle_id,omitempty"`
-}
-
-func (m *VolumeSyncDataRequest) Reset() { *m = VolumeSyncDataRequest{} }
-func (m *VolumeSyncDataRequest) String() string { return proto.CompactTextString(m) }
-func (*VolumeSyncDataRequest) ProtoMessage() {}
-func (*VolumeSyncDataRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
-
-func (m *VolumeSyncDataRequest) GetVolumeId() uint32 {
- if m != nil {
- return m.VolumeId
- }
- return 0
-}
-
-func (m *VolumeSyncDataRequest) GetRevision() uint32 {
- if m != nil {
- return m.Revision
- }
- return 0
-}
-
-func (m *VolumeSyncDataRequest) GetOffset() uint32 {
- if m != nil {
- return m.Offset
- }
- return 0
-}
-
-func (m *VolumeSyncDataRequest) GetSize() uint32 {
+func (m *VolumeFollowRequest) GetSince() uint64 {
if m != nil {
- return m.Size
+ return m.Since
}
return 0
}
-func (m *VolumeSyncDataRequest) GetNeedleId() string {
- if m != nil {
- return m.NeedleId
- }
- return ""
-}
-
-type VolumeSyncDataResponse struct {
+type VolumeFollowResponse struct {
FileContent []byte `protobuf:"bytes,1,opt,name=file_content,json=fileContent,proto3" json:"file_content,omitempty"`
}
-func (m *VolumeSyncDataResponse) Reset() { *m = VolumeSyncDataResponse{} }
-func (m *VolumeSyncDataResponse) String() string { return proto.CompactTextString(m) }
-func (*VolumeSyncDataResponse) ProtoMessage() {}
-func (*VolumeSyncDataResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
+func (m *VolumeFollowResponse) Reset() { *m = VolumeFollowResponse{} }
+func (m *VolumeFollowResponse) String() string { return proto.CompactTextString(m) }
+func (*VolumeFollowResponse) ProtoMessage() {}
+func (*VolumeFollowResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
-func (m *VolumeSyncDataResponse) GetFileContent() []byte {
+func (m *VolumeFollowResponse) GetFileContent() []byte {
if m != nil {
return m.FileContent
}
@@ -523,7 +465,7 @@ type VolumeMountRequest struct {
func (m *VolumeMountRequest) Reset() { *m = VolumeMountRequest{} }
func (m *VolumeMountRequest) String() string { return proto.CompactTextString(m) }
func (*VolumeMountRequest) ProtoMessage() {}
-func (*VolumeMountRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
+func (*VolumeMountRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
func (m *VolumeMountRequest) GetVolumeId() uint32 {
if m != nil {
@@ -538,7 +480,7 @@ type VolumeMountResponse struct {
func (m *VolumeMountResponse) Reset() { *m = VolumeMountResponse{} }
func (m *VolumeMountResponse) String() string { return proto.CompactTextString(m) }
func (*VolumeMountResponse) ProtoMessage() {}
-func (*VolumeMountResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
+func (*VolumeMountResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
type VolumeUnmountRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
@@ -547,7 +489,7 @@ type VolumeUnmountRequest struct {
func (m *VolumeUnmountRequest) Reset() { *m = VolumeUnmountRequest{} }
func (m *VolumeUnmountRequest) String() string { return proto.CompactTextString(m) }
func (*VolumeUnmountRequest) ProtoMessage() {}
-func (*VolumeUnmountRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} }
+func (*VolumeUnmountRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
func (m *VolumeUnmountRequest) GetVolumeId() uint32 {
if m != nil {
@@ -562,7 +504,7 @@ type VolumeUnmountResponse struct {
func (m *VolumeUnmountResponse) Reset() { *m = VolumeUnmountResponse{} }
func (m *VolumeUnmountResponse) String() string { return proto.CompactTextString(m) }
func (*VolumeUnmountResponse) ProtoMessage() {}
-func (*VolumeUnmountResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
+func (*VolumeUnmountResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
type VolumeDeleteRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
@@ -571,7 +513,7 @@ type VolumeDeleteRequest struct {
func (m *VolumeDeleteRequest) Reset() { *m = VolumeDeleteRequest{} }
func (m *VolumeDeleteRequest) String() string { return proto.CompactTextString(m) }
func (*VolumeDeleteRequest) ProtoMessage() {}
-func (*VolumeDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} }
+func (*VolumeDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} }
func (m *VolumeDeleteRequest) GetVolumeId() uint32 {
if m != nil {
@@ -586,7 +528,7 @@ type VolumeDeleteResponse struct {
func (m *VolumeDeleteResponse) Reset() { *m = VolumeDeleteResponse{} }
func (m *VolumeDeleteResponse) String() string { return proto.CompactTextString(m) }
func (*VolumeDeleteResponse) ProtoMessage() {}
-func (*VolumeDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27} }
+func (*VolumeDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
type ReplicateVolumeRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
@@ -599,7 +541,7 @@ type ReplicateVolumeRequest struct {
func (m *ReplicateVolumeRequest) Reset() { *m = ReplicateVolumeRequest{} }
func (m *ReplicateVolumeRequest) String() string { return proto.CompactTextString(m) }
func (*ReplicateVolumeRequest) ProtoMessage() {}
-func (*ReplicateVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28} }
+func (*ReplicateVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} }
func (m *ReplicateVolumeRequest) GetVolumeId() uint32 {
if m != nil {
@@ -642,7 +584,7 @@ type ReplicateVolumeResponse struct {
func (m *ReplicateVolumeResponse) Reset() { *m = ReplicateVolumeResponse{} }
func (m *ReplicateVolumeResponse) String() string { return proto.CompactTextString(m) }
func (*ReplicateVolumeResponse) ProtoMessage() {}
-func (*ReplicateVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{29} }
+func (*ReplicateVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27} }
type CopyFileRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
@@ -653,7 +595,7 @@ type CopyFileRequest struct {
func (m *CopyFileRequest) Reset() { *m = CopyFileRequest{} }
func (m *CopyFileRequest) String() string { return proto.CompactTextString(m) }
func (*CopyFileRequest) ProtoMessage() {}
-func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{30} }
+func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28} }
func (m *CopyFileRequest) GetVolumeId() uint32 {
if m != nil {
@@ -683,7 +625,7 @@ type CopyFileResponse struct {
func (m *CopyFileResponse) Reset() { *m = CopyFileResponse{} }
func (m *CopyFileResponse) String() string { return proto.CompactTextString(m) }
func (*CopyFileResponse) ProtoMessage() {}
-func (*CopyFileResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{31} }
+func (*CopyFileResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{29} }
func (m *CopyFileResponse) GetFileContent() []byte {
if m != nil {
@@ -699,7 +641,7 @@ type ReadVolumeFileStatusRequest struct {
func (m *ReadVolumeFileStatusRequest) Reset() { *m = ReadVolumeFileStatusRequest{} }
func (m *ReadVolumeFileStatusRequest) String() string { return proto.CompactTextString(m) }
func (*ReadVolumeFileStatusRequest) ProtoMessage() {}
-func (*ReadVolumeFileStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{32} }
+func (*ReadVolumeFileStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{30} }
func (m *ReadVolumeFileStatusRequest) GetVolumeId() uint32 {
if m != nil {
@@ -719,7 +661,7 @@ type ReadVolumeFileStatusResponse struct {
func (m *ReadVolumeFileStatusResponse) Reset() { *m = ReadVolumeFileStatusResponse{} }
func (m *ReadVolumeFileStatusResponse) String() string { return proto.CompactTextString(m) }
func (*ReadVolumeFileStatusResponse) ProtoMessage() {}
-func (*ReadVolumeFileStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{33} }
+func (*ReadVolumeFileStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{31} }
func (m *ReadVolumeFileStatusResponse) GetVolumeId() uint32 {
if m != nil {
@@ -766,7 +708,7 @@ type DiskStatus struct {
func (m *DiskStatus) Reset() { *m = DiskStatus{} }
func (m *DiskStatus) String() string { return proto.CompactTextString(m) }
func (*DiskStatus) ProtoMessage() {}
-func (*DiskStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{34} }
+func (*DiskStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{32} }
func (m *DiskStatus) GetDir() string {
if m != nil {
@@ -809,7 +751,7 @@ type MemStatus struct {
func (m *MemStatus) Reset() { *m = MemStatus{} }
func (m *MemStatus) String() string { return proto.CompactTextString(m) }
func (*MemStatus) ProtoMessage() {}
-func (*MemStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{35} }
+func (*MemStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{33} }
func (m *MemStatus) GetGoroutines() int32 {
if m != nil {
@@ -879,10 +821,8 @@ func init() {
proto.RegisterType((*AssignVolumeResponse)(nil), "volume_server_pb.AssignVolumeResponse")
proto.RegisterType((*VolumeSyncStatusRequest)(nil), "volume_server_pb.VolumeSyncStatusRequest")
proto.RegisterType((*VolumeSyncStatusResponse)(nil), "volume_server_pb.VolumeSyncStatusResponse")
- proto.RegisterType((*VolumeSyncIndexRequest)(nil), "volume_server_pb.VolumeSyncIndexRequest")
- proto.RegisterType((*VolumeSyncIndexResponse)(nil), "volume_server_pb.VolumeSyncIndexResponse")
- proto.RegisterType((*VolumeSyncDataRequest)(nil), "volume_server_pb.VolumeSyncDataRequest")
- proto.RegisterType((*VolumeSyncDataResponse)(nil), "volume_server_pb.VolumeSyncDataResponse")
+ proto.RegisterType((*VolumeFollowRequest)(nil), "volume_server_pb.VolumeFollowRequest")
+ proto.RegisterType((*VolumeFollowResponse)(nil), "volume_server_pb.VolumeFollowResponse")
proto.RegisterType((*VolumeMountRequest)(nil), "volume_server_pb.VolumeMountRequest")
proto.RegisterType((*VolumeMountResponse)(nil), "volume_server_pb.VolumeMountResponse")
proto.RegisterType((*VolumeUnmountRequest)(nil), "volume_server_pb.VolumeUnmountRequest")
@@ -919,8 +859,7 @@ type VolumeServerClient interface {
DeleteCollection(ctx context.Context, in *DeleteCollectionRequest, opts ...grpc.CallOption) (*DeleteCollectionResponse, error)
AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error)
VolumeSyncStatus(ctx context.Context, in *VolumeSyncStatusRequest, opts ...grpc.CallOption) (*VolumeSyncStatusResponse, error)
- VolumeSyncIndex(ctx context.Context, in *VolumeSyncIndexRequest, opts ...grpc.CallOption) (VolumeServer_VolumeSyncIndexClient, error)
- VolumeSyncData(ctx context.Context, in *VolumeSyncDataRequest, opts ...grpc.CallOption) (VolumeServer_VolumeSyncDataClient, error)
+ VolumeFollow(ctx context.Context, in *VolumeFollowRequest, opts ...grpc.CallOption) (VolumeServer_VolumeFollowClient, error)
VolumeMount(ctx context.Context, in *VolumeMountRequest, opts ...grpc.CallOption) (*VolumeMountResponse, error)
VolumeUnmount(ctx context.Context, in *VolumeUnmountRequest, opts ...grpc.CallOption) (*VolumeUnmountResponse, error)
VolumeDelete(ctx context.Context, in *VolumeDeleteRequest, opts ...grpc.CallOption) (*VolumeDeleteResponse, error)
@@ -1009,12 +948,12 @@ func (c *volumeServerClient) VolumeSyncStatus(ctx context.Context, in *VolumeSyn
return out, nil
}
-func (c *volumeServerClient) VolumeSyncIndex(ctx context.Context, in *VolumeSyncIndexRequest, opts ...grpc.CallOption) (VolumeServer_VolumeSyncIndexClient, error) {
- stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[0], c.cc, "/volume_server_pb.VolumeServer/VolumeSyncIndex", opts...)
+func (c *volumeServerClient) VolumeFollow(ctx context.Context, in *VolumeFollowRequest, opts ...grpc.CallOption) (VolumeServer_VolumeFollowClient, error) {
+ stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[0], c.cc, "/volume_server_pb.VolumeServer/VolumeFollow", opts...)
if err != nil {
return nil, err
}
- x := &volumeServerVolumeSyncIndexClient{stream}
+ x := &volumeServerVolumeFollowClient{stream}
if err := x.ClientStream.SendMsg(in); err != nil {
return nil, err
}
@@ -1024,49 +963,17 @@ func (c *volumeServerClient) VolumeSyncIndex(ctx context.Context, in *VolumeSync
return x, nil
}
-type VolumeServer_VolumeSyncIndexClient interface {
- Recv() (*VolumeSyncIndexResponse, error)
+type VolumeServer_VolumeFollowClient interface {
+ Recv() (*VolumeFollowResponse, error)
grpc.ClientStream
}
-type volumeServerVolumeSyncIndexClient struct {
+type volumeServerVolumeFollowClient struct {
grpc.ClientStream
}
-func (x *volumeServerVolumeSyncIndexClient) Recv() (*VolumeSyncIndexResponse, error) {
- m := new(VolumeSyncIndexResponse)
- if err := x.ClientStream.RecvMsg(m); err != nil {
- return nil, err
- }
- return m, nil
-}
-
-func (c *volumeServerClient) VolumeSyncData(ctx context.Context, in *VolumeSyncDataRequest, opts ...grpc.CallOption) (VolumeServer_VolumeSyncDataClient, error) {
- stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[1], c.cc, "/volume_server_pb.VolumeServer/VolumeSyncData", opts...)
- if err != nil {
- return nil, err
- }
- x := &volumeServerVolumeSyncDataClient{stream}
- if err := x.ClientStream.SendMsg(in); err != nil {
- return nil, err
- }
- if err := x.ClientStream.CloseSend(); err != nil {
- return nil, err
- }
- return x, nil
-}
-
-type VolumeServer_VolumeSyncDataClient interface {
- Recv() (*VolumeSyncDataResponse, error)
- grpc.ClientStream
-}
-
-type volumeServerVolumeSyncDataClient struct {
- grpc.ClientStream
-}
-
-func (x *volumeServerVolumeSyncDataClient) Recv() (*VolumeSyncDataResponse, error) {
- m := new(VolumeSyncDataResponse)
+func (x *volumeServerVolumeFollowClient) Recv() (*VolumeFollowResponse, error) {
+ m := new(VolumeFollowResponse)
if err := x.ClientStream.RecvMsg(m); err != nil {
return nil, err
}
@@ -1119,7 +1026,7 @@ func (c *volumeServerClient) ReadVolumeFileStatus(ctx context.Context, in *ReadV
}
func (c *volumeServerClient) CopyFile(ctx context.Context, in *CopyFileRequest, opts ...grpc.CallOption) (VolumeServer_CopyFileClient, error) {
- stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[2], c.cc, "/volume_server_pb.VolumeServer/CopyFile", opts...)
+ stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[1], c.cc, "/volume_server_pb.VolumeServer/CopyFile", opts...)
if err != nil {
return nil, err
}
@@ -1162,8 +1069,7 @@ type VolumeServerServer interface {
DeleteCollection(context.Context, *DeleteCollectionRequest) (*DeleteCollectionResponse, error)
AssignVolume(context.Context, *AssignVolumeRequest) (*AssignVolumeResponse, error)
VolumeSyncStatus(context.Context, *VolumeSyncStatusRequest) (*VolumeSyncStatusResponse, error)
- VolumeSyncIndex(*VolumeSyncIndexRequest, VolumeServer_VolumeSyncIndexServer) error
- VolumeSyncData(*VolumeSyncDataRequest, VolumeServer_VolumeSyncDataServer) error
+ VolumeFollow(*VolumeFollowRequest, VolumeServer_VolumeFollowServer) error
VolumeMount(context.Context, *VolumeMountRequest) (*VolumeMountResponse, error)
VolumeUnmount(context.Context, *VolumeUnmountRequest) (*VolumeUnmountResponse, error)
VolumeDelete(context.Context, *VolumeDeleteRequest) (*VolumeDeleteResponse, error)
@@ -1320,45 +1226,24 @@ func _VolumeServer_VolumeSyncStatus_Handler(srv interface{}, ctx context.Context
return interceptor(ctx, in, info, handler)
}
-func _VolumeServer_VolumeSyncIndex_Handler(srv interface{}, stream grpc.ServerStream) error {
- m := new(VolumeSyncIndexRequest)
- if err := stream.RecvMsg(m); err != nil {
- return err
- }
- return srv.(VolumeServerServer).VolumeSyncIndex(m, &volumeServerVolumeSyncIndexServer{stream})
-}
-
-type VolumeServer_VolumeSyncIndexServer interface {
- Send(*VolumeSyncIndexResponse) error
- grpc.ServerStream
-}
-
-type volumeServerVolumeSyncIndexServer struct {
- grpc.ServerStream
-}
-
-func (x *volumeServerVolumeSyncIndexServer) Send(m *VolumeSyncIndexResponse) error {
- return x.ServerStream.SendMsg(m)
-}
-
-func _VolumeServer_VolumeSyncData_Handler(srv interface{}, stream grpc.ServerStream) error {
- m := new(VolumeSyncDataRequest)
+func _VolumeServer_VolumeFollow_Handler(srv interface{}, stream grpc.ServerStream) error {
+ m := new(VolumeFollowRequest)
if err := stream.RecvMsg(m); err != nil {
return err
}
- return srv.(VolumeServerServer).VolumeSyncData(m, &volumeServerVolumeSyncDataServer{stream})
+ return srv.(VolumeServerServer).VolumeFollow(m, &volumeServerVolumeFollowServer{stream})
}
-type VolumeServer_VolumeSyncDataServer interface {
- Send(*VolumeSyncDataResponse) error
+type VolumeServer_VolumeFollowServer interface {
+ Send(*VolumeFollowResponse) error
grpc.ServerStream
}
-type volumeServerVolumeSyncDataServer struct {
+type volumeServerVolumeFollowServer struct {
grpc.ServerStream
}
-func (x *volumeServerVolumeSyncDataServer) Send(m *VolumeSyncDataResponse) error {
+func (x *volumeServerVolumeFollowServer) Send(m *VolumeFollowResponse) error {
return x.ServerStream.SendMsg(m)
}
@@ -1532,13 +1417,8 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{
},
Streams: []grpc.StreamDesc{
{
- StreamName: "VolumeSyncIndex",
- Handler: _VolumeServer_VolumeSyncIndex_Handler,
- ServerStreams: true,
- },
- {
- StreamName: "VolumeSyncData",
- Handler: _VolumeServer_VolumeSyncData_Handler,
+ StreamName: "VolumeFollow",
+ Handler: _VolumeServer_VolumeFollow_Handler,
ServerStreams: true,
},
{
@@ -1553,83 +1433,79 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{
func init() { proto.RegisterFile("volume_server.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
- // 1247 bytes of a gzipped FileDescriptorProto
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x58, 0x5b, 0x73, 0xdb, 0xc4,
- 0x17, 0x8f, 0x62, 0x3b, 0x71, 0x8e, 0xed, 0xc6, 0xff, 0x4d, 0x9a, 0x38, 0x4a, 0xff, 0xc1, 0x15,
- 0x90, 0x3a, 0x6d, 0x1a, 0x20, 0x9d, 0x42, 0x81, 0x17, 0x20, 0x01, 0x26, 0x0f, 0xa5, 0x33, 0x0a,
- 0xed, 0x30, 0x43, 0x67, 0x34, 0x1b, 0x69, 0x9d, 0x88, 0xc8, 0x92, 0xaa, 0x5d, 0x85, 0x84, 0x6f,
- 0xc2, 0x33, 0x2f, 0x7d, 0xe7, 0x03, 0xf1, 0x41, 0x78, 0x61, 0xf6, 0x22, 0x59, 0x37, 0xc7, 0xe2,
- 0xf2, 0xb6, 0x3a, 0x7b, 0xce, 0xef, 0x5c, 0xf6, 0xec, 0xd9, 0x9f, 0x0d, 0x6b, 0x57, 0x81, 0x17,
- 0x4f, 0x88, 0x45, 0x49, 0x74, 0x45, 0xa2, 0x83, 0x30, 0x0a, 0x58, 0x80, 0xfa, 0x39, 0xa1, 0x15,
- 0x9e, 0x19, 0x1f, 0x00, 0xfa, 0x0a, 0x33, 0xfb, 0xe2, 0x98, 0x78, 0x84, 0x11, 0x93, 0xbc, 0x89,
- 0x09, 0x65, 0x68, 0x0b, 0xda, 0x63, 0xd7, 0x23, 0x96, 0xeb, 0xd0, 0x81, 0x36, 0x6c, 0x8c, 0x56,
- 0xcc, 0x65, 0xfe, 0x7d, 0xe2, 0x50, 0xe3, 0x05, 0xac, 0xe5, 0x0c, 0x68, 0x18, 0xf8, 0x94, 0xa0,
- 0x67, 0xb0, 0x1c, 0x11, 0x1a, 0x7b, 0x4c, 0x1a, 0x74, 0x0e, 0x77, 0x0e, 0x8a, 0xbe, 0x0e, 0x52,
- 0x93, 0xd8, 0x63, 0x66, 0xa2, 0x6e, 0xb8, 0xd0, 0xcd, 0x6e, 0xa0, 0x4d, 0x58, 0x56, 0xbe, 0x07,
- 0xda, 0x50, 0x1b, 0xad, 0x98, 0x4b, 0xd2, 0x35, 0xda, 0x80, 0x25, 0xca, 0x30, 0x8b, 0xe9, 0x60,
- 0x71, 0xa8, 0x8d, 0x5a, 0xa6, 0xfa, 0x42, 0xeb, 0xd0, 0x22, 0x51, 0x14, 0x44, 0x83, 0x86, 0x50,
- 0x97, 0x1f, 0x08, 0x41, 0x93, 0xba, 0xbf, 0x90, 0x41, 0x73, 0xa8, 0x8d, 0x7a, 0xa6, 0x58, 0x1b,
- 0xcb, 0xd0, 0xfa, 0x7a, 0x12, 0xb2, 0x1b, 0xe3, 0x13, 0x18, 0xbc, 0xc2, 0x76, 0x1c, 0x4f, 0x5e,
- 0x89, 0x18, 0x8f, 0x2e, 0x88, 0x7d, 0x99, 0xe4, 0xbe, 0x0d, 0x2b, 0x2a, 0x72, 0x15, 0x41, 0xcf,
- 0x6c, 0x4b, 0xc1, 0x89, 0x63, 0x7c, 0x01, 0x5b, 0x15, 0x86, 0xaa, 0x06, 0xef, 0x42, 0xef, 0x1c,
- 0x47, 0x67, 0xf8, 0x9c, 0x58, 0x11, 0x66, 0x6e, 0x20, 0xac, 0x35, 0xb3, 0xab, 0x84, 0x26, 0x97,
- 0x19, 0x3f, 0x82, 0x9e, 0x43, 0x08, 0x26, 0x21, 0xb6, 0x59, 0x1d, 0xe7, 0x68, 0x08, 0x9d, 0x30,
- 0x22, 0xd8, 0xf3, 0x02, 0x1b, 0x33, 0x22, 0xaa, 0xd0, 0x30, 0xb3, 0x22, 0xe3, 0xff, 0xb0, 0x5d,
- 0x09, 0x2e, 0x03, 0x34, 0x9e, 0x15, 0xa2, 0x0f, 0x26, 0x13, 0xb7, 0x96, 0x6b, 0xe3, 0x5e, 0x29,
- 0x6a, 0x61, 0xa9, 0x70, 0x3f, 0x2d, 0xec, 0x7a, 0x04, 0xfb, 0x71, 0x58, 0x0b, 0xb8, 0x18, 0x71,
- 0x62, 0x9a, 0x22, 0x6f, 0xca, 0xe6, 0x38, 0x0a, 0x3c, 0x8f, 0xd8, 0xcc, 0x0d, 0xfc, 0x04, 0x76,
- 0x07, 0xc0, 0x4e, 0x85, 0xaa, 0x55, 0x32, 0x12, 0x43, 0x87, 0x41, 0xd9, 0x54, 0xc1, 0xbe, 0xd5,
- 0x60, 0xed, 0x4b, 0x4a, 0xdd, 0x73, 0x5f, 0xba, 0xad, 0x55, 0xfe, 0xbc, 0xc3, 0xc5, 0xa2, 0xc3,
- 0xe2, 0xf1, 0x34, 0x4a, 0xc7, 0xc3, 0x35, 0x22, 0x12, 0x7a, 0xae, 0x8d, 0x05, 0x44, 0x53, 0x40,
- 0x64, 0x45, 0xa8, 0x0f, 0x0d, 0xc6, 0xbc, 0x41, 0x4b, 0xec, 0xf0, 0xa5, 0xb1, 0x01, 0xeb, 0xf9,
- 0x48, 0x55, 0x0a, 0x1f, 0xc3, 0xa6, 0x94, 0x9c, 0xde, 0xf8, 0xf6, 0xa9, 0xb8, 0x09, 0xb5, 0x0a,
- 0xfe, 0xa7, 0x06, 0x83, 0xb2, 0xa1, 0xea, 0xe0, 0x7f, 0x9b, 0xff, 0xdf, 0xcd, 0x0e, 0xbd, 0x03,
- 0x1d, 0x86, 0x5d, 0xcf, 0x0a, 0xc6, 0x63, 0x4a, 0xd8, 0x60, 0x69, 0xa8, 0x8d, 0x9a, 0x26, 0x70,
- 0xd1, 0x0b, 0x21, 0x41, 0x7b, 0xd0, 0xb7, 0x65, 0x17, 0x5b, 0x11, 0xb9, 0x72, 0x29, 0x47, 0x5e,
- 0x16, 0x81, 0xad, 0xda, 0x49, 0x77, 0x4b, 0x31, 0x32, 0xa0, 0xe7, 0x3a, 0xd7, 0x96, 0x18, 0x1e,
- 0xe2, 0xea, 0xb7, 0x05, 0x5a, 0xc7, 0x75, 0xae, 0xbf, 0x71, 0x3d, 0x72, 0xca, 0x27, 0xc0, 0x53,
- 0xd8, 0x98, 0x26, 0x7f, 0xe2, 0x3b, 0xe4, 0xba, 0x56, 0xd1, 0xbe, 0xcd, 0x16, 0x5b, 0x99, 0xa9,
- 0x92, 0xed, 0x03, 0x72, 0xb9, 0x40, 0xfa, 0xb5, 0x03, 0x9f, 0x11, 0x9f, 0x09, 0x80, 0xae, 0xd9,
- 0x17, 0x3b, 0xdc, 0xf9, 0x91, 0x94, 0x1b, 0xbf, 0x6a, 0x70, 0x77, 0x8a, 0x74, 0x8c, 0x19, 0xae,
- 0xd5, 0x7a, 0x3a, 0xb4, 0xd3, 0xec, 0x17, 0xe5, 0x5e, 0xf2, 0xcd, 0xc7, 0xa2, 0xaa, 0x5e, 0x43,
- 0xec, 0xa8, 0xaf, 0xaa, 0x01, 0xc8, 0x9d, 0xf8, 0x84, 0x38, 0x72, 0xba, 0xca, 0x63, 0x68, 0x4b,
- 0xc1, 0x89, 0x63, 0x7c, 0x9e, 0xad, 0x8d, 0x0c, 0x4d, 0xe5, 0x78, 0x1f, 0xba, 0x15, 0xd9, 0x75,
- 0xc6, 0x99, 0xc4, 0x3e, 0x02, 0x24, 0x8d, 0x9f, 0x07, 0xb1, 0x5f, 0x6f, 0xa6, 0xdc, 0x85, 0xb5,
- 0x9c, 0x89, 0x6a, 0xec, 0x27, 0xb0, 0x2e, 0xc5, 0x2f, 0xfd, 0x49, 0x6d, 0xac, 0xcd, 0xa4, 0xac,
- 0xa9, 0x91, 0x42, 0x3b, 0x4c, 0x9c, 0xe4, 0x1f, 0xb8, 0x5b, 0xc1, 0x36, 0x92, 0x08, 0xf2, 0x6f,
- 0x9c, 0xf1, 0xbb, 0x06, 0x1b, 0xa6, 0x6a, 0x67, 0xf2, 0xdf, 0x0e, 0x8e, 0xec, 0xc5, 0x69, 0xcc,
- 0xbc, 0x38, 0xcd, 0xe9, 0xc5, 0x19, 0x41, 0x9f, 0x06, 0x71, 0x64, 0x13, 0xcb, 0xc1, 0x0c, 0x5b,
- 0x7e, 0xe0, 0x10, 0x75, 0xa0, 0x77, 0xa4, 0x9c, 0x1f, 0xe0, 0x77, 0x81, 0x43, 0x8c, 0x2d, 0xd8,
- 0x2c, 0x05, 0xad, 0x12, 0xf2, 0x61, 0xf5, 0x28, 0x08, 0x6f, 0x78, 0x83, 0xd6, 0x4c, 0xa4, 0xe3,
- 0x52, 0x2b, 0xb9, 0x64, 0x22, 0x93, 0xb6, 0xb9, 0xe2, 0xd2, 0x13, 0x79, 0xc3, 0xd4, 0xbe, 0x83,
- 0x99, 0xdc, 0x6f, 0x24, 0xfb, 0xc7, 0x98, 0xf1, 0x7d, 0xe3, 0x29, 0xf4, 0xa7, 0xfe, 0xea, 0xf7,
- 0xd6, 0x67, 0xb0, 0x6d, 0x12, 0xec, 0xc8, 0xe0, 0xc5, 0x55, 0xae, 0x3f, 0xee, 0xfe, 0xd0, 0xe0,
- 0x5e, 0xb5, 0x71, 0x9d, 0x91, 0xc7, 0x2f, 0x77, 0x32, 0x52, 0x98, 0x3b, 0x21, 0x94, 0xe1, 0x49,
- 0x28, 0xf2, 0x6e, 0x9a, 0x7d, 0x35, 0x57, 0xbe, 0x4f, 0xe4, 0xe5, 0x01, 0xd4, 0x28, 0x0d, 0x20,
- 0x8e, 0x98, 0xd4, 0x27, 0x83, 0xd8, 0x94, 0x88, 0x8e, 0xac, 0x53, 0x0e, 0x31, 0xd5, 0x16, 0x88,
- 0x2d, 0x89, 0xa8, 0x14, 0xc5, 0x48, 0xfb, 0x01, 0xe0, 0xd8, 0xa5, 0x97, 0x32, 0x2d, 0xde, 0x29,
- 0x8e, 0x1b, 0xa9, 0xe7, 0x90, 0x2f, 0xb9, 0x04, 0x7b, 0x9e, 0x0a, 0x9a, 0x2f, 0xf9, 0x64, 0x88,
- 0x29, 0x71, 0x54, 0x78, 0x62, 0xcd, 0x65, 0xe3, 0x88, 0x10, 0x15, 0x89, 0x58, 0x1b, 0xbf, 0x69,
- 0xb0, 0xf2, 0x9c, 0x4c, 0x14, 0xf2, 0x0e, 0xc0, 0x79, 0x10, 0x05, 0x31, 0x73, 0x7d, 0x42, 0x85,
- 0x83, 0x96, 0x99, 0x91, 0xfc, 0x73, 0x3f, 0x62, 0x52, 0x11, 0x6f, 0xac, 0x92, 0x13, 0x6b, 0x2e,
- 0xbb, 0x20, 0x38, 0x54, 0x2f, 0x82, 0x58, 0x73, 0xa2, 0x47, 0x19, 0xb6, 0x2f, 0xc5, 0x03, 0xd0,
- 0x34, 0xe5, 0xc7, 0xe1, 0xdb, 0x1e, 0x74, 0xd5, 0xdc, 0x12, 0x4c, 0x13, 0xbd, 0x86, 0x4e, 0x86,
- 0xa1, 0xa2, 0xf7, 0xca, 0x44, 0xb4, 0xcc, 0x78, 0xf5, 0xf7, 0xe7, 0x68, 0xa9, 0x1b, 0xb3, 0x80,
- 0x7c, 0xf8, 0x5f, 0x89, 0x01, 0xa2, 0x87, 0x65, 0xeb, 0x59, 0xfc, 0x52, 0x7f, 0x54, 0x4b, 0x37,
- 0xf5, 0xc7, 0x60, 0xad, 0x82, 0xd2, 0xa1, 0xfd, 0x39, 0x28, 0x39, 0x5a, 0xa9, 0x3f, 0xae, 0xa9,
- 0x9d, 0x7a, 0x7d, 0x03, 0xa8, 0xcc, 0xf7, 0xd0, 0xa3, 0xb9, 0x30, 0x53, 0x3e, 0xa9, 0xef, 0xd7,
- 0x53, 0x9e, 0x99, 0xa8, 0x64, 0x82, 0x73, 0x13, 0xcd, 0x71, 0xcd, 0xb9, 0x89, 0x16, 0xe8, 0xe5,
- 0x02, 0xba, 0x84, 0x7e, 0x91, 0x25, 0xa2, 0xbd, 0x59, 0x3f, 0x5d, 0x4a, 0x24, 0x54, 0x7f, 0x58,
- 0x47, 0x35, 0x75, 0x66, 0x41, 0x37, 0xcb, 0xe5, 0x50, 0x45, 0xd3, 0x55, 0xb0, 0x52, 0x7d, 0x77,
- 0x9e, 0x5a, 0x36, 0x9b, 0x22, 0xb7, 0xab, 0xca, 0x66, 0x06, 0x71, 0xac, 0xca, 0x66, 0x16, 0x55,
- 0x34, 0x16, 0xd0, 0x4f, 0xb0, 0x5a, 0x20, 0x45, 0x68, 0x74, 0x1b, 0x40, 0x96, 0x6e, 0xe9, 0x7b,
- 0x35, 0x34, 0x13, 0x4f, 0x1f, 0x6a, 0xe8, 0x1c, 0xee, 0xe4, 0xb9, 0x09, 0x7a, 0x70, 0x1b, 0x40,
- 0x86, 0x58, 0xe9, 0xa3, 0xf9, 0x8a, 0x19, 0x47, 0xaf, 0xa1, 0x93, 0x21, 0x25, 0x55, 0xc3, 0xa3,
- 0x4c, 0x73, 0xaa, 0x86, 0x47, 0x15, 0xb3, 0x59, 0x40, 0x67, 0xd0, 0xcb, 0xd1, 0x14, 0xb4, 0x3b,
- 0xcb, 0x32, 0x4f, 0x7e, 0xf4, 0x07, 0x73, 0xf5, 0xb2, 0x4d, 0x96, 0x65, 0x2f, 0x68, 0x66, 0x70,
- 0xf9, 0x01, 0xb8, 0x3b, 0x4f, 0x2d, 0x75, 0x70, 0x01, 0xab, 0x05, 0x42, 0x51, 0x75, 0xee, 0xd5,
- 0x44, 0xa9, 0xea, 0xdc, 0x67, 0xb1, 0x93, 0x05, 0xf4, 0x33, 0xac, 0x57, 0xbd, 0xdd, 0xe8, 0x71,
- 0x15, 0xc8, 0x4c, 0x82, 0xa0, 0x1f, 0xd4, 0x55, 0x4f, 0x1d, 0xbf, 0x84, 0x76, 0x42, 0x54, 0xd0,
- 0xfd, 0xb2, 0x75, 0x81, 0x34, 0xe9, 0xc6, 0x6d, 0x2a, 0xd3, 0xe6, 0x3a, 0x5b, 0x12, 0xff, 0xc2,
- 0x3c, 0xf9, 0x2b, 0x00, 0x00, 0xff, 0xff, 0x6c, 0x7d, 0x0d, 0xbb, 0x9c, 0x11, 0x00, 0x00,
+ // 1169 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x58, 0xdd, 0x6f, 0xdc, 0x44,
+ 0x10, 0x8f, 0x7b, 0x77, 0xf9, 0x98, 0xbb, 0xd0, 0x63, 0x93, 0x26, 0x57, 0xb7, 0x84, 0xeb, 0x42,
+ 0xcb, 0xa5, 0x4d, 0x03, 0xa4, 0x02, 0x5a, 0x9e, 0x80, 0x84, 0x8a, 0x3c, 0x94, 0x4a, 0x0e, 0xad,
+ 0x90, 0x40, 0xb2, 0x36, 0xf6, 0x26, 0xb1, 0xe2, 0xf3, 0xba, 0xde, 0x75, 0xda, 0xf0, 0xef, 0xf0,
+ 0xc2, 0x3b, 0x12, 0xff, 0x0e, 0x7f, 0x08, 0x2f, 0x68, 0x3f, 0xec, 0xf8, 0xeb, 0x7a, 0xe6, 0xe3,
+ 0x6d, 0x3d, 0x3b, 0xf3, 0x9b, 0x99, 0xdd, 0x99, 0xd9, 0x9f, 0x0c, 0x6b, 0x17, 0x2c, 0x4c, 0xa7,
+ 0xd4, 0xe5, 0x34, 0xb9, 0xa0, 0xc9, 0x6e, 0x9c, 0x30, 0xc1, 0xd0, 0xb0, 0x24, 0x74, 0xe3, 0x63,
+ 0xfc, 0x31, 0xa0, 0x6f, 0x88, 0xf0, 0xce, 0x0e, 0x68, 0x48, 0x05, 0x75, 0xe8, 0xab, 0x94, 0x72,
+ 0x81, 0x6e, 0xc2, 0xf2, 0x49, 0x10, 0x52, 0x37, 0xf0, 0xf9, 0xc8, 0x1a, 0x77, 0x26, 0x2b, 0xce,
+ 0x92, 0xfc, 0x3e, 0xf4, 0x39, 0x7e, 0x0e, 0x6b, 0x25, 0x03, 0x1e, 0xb3, 0x88, 0x53, 0xf4, 0x18,
+ 0x96, 0x12, 0xca, 0xd3, 0x50, 0x68, 0x83, 0xfe, 0xde, 0xd6, 0x6e, 0xd5, 0xd7, 0x6e, 0x6e, 0x92,
+ 0x86, 0xc2, 0xc9, 0xd4, 0x71, 0x00, 0x83, 0xe2, 0x06, 0xda, 0x84, 0x25, 0xe3, 0x7b, 0x64, 0x8d,
+ 0xad, 0xc9, 0x8a, 0xb3, 0xa8, 0x5d, 0xa3, 0x0d, 0x58, 0xe4, 0x82, 0x88, 0x94, 0x8f, 0xae, 0x8d,
+ 0xad, 0x49, 0xcf, 0x31, 0x5f, 0x68, 0x1d, 0x7a, 0x34, 0x49, 0x58, 0x32, 0xea, 0x28, 0x75, 0xfd,
+ 0x81, 0x10, 0x74, 0x79, 0xf0, 0x0b, 0x1d, 0x75, 0xc7, 0xd6, 0x64, 0xd5, 0x51, 0x6b, 0xbc, 0x04,
+ 0xbd, 0x6f, 0xa7, 0xb1, 0xb8, 0xc4, 0x5f, 0xc0, 0xe8, 0x25, 0xf1, 0xd2, 0x74, 0xfa, 0x52, 0xc5,
+ 0xb8, 0x7f, 0x46, 0xbd, 0xf3, 0x2c, 0xf7, 0x5b, 0xb0, 0x62, 0x22, 0x37, 0x11, 0xac, 0x3a, 0xcb,
+ 0x5a, 0x70, 0xe8, 0xe3, 0xaf, 0xe0, 0x66, 0x83, 0xa1, 0x39, 0x83, 0x0f, 0x60, 0xf5, 0x94, 0x24,
+ 0xc7, 0xe4, 0x94, 0xba, 0x09, 0x11, 0x01, 0x53, 0xd6, 0x96, 0x33, 0x30, 0x42, 0x47, 0xca, 0xf0,
+ 0x4f, 0x60, 0x97, 0x10, 0xd8, 0x34, 0x26, 0x9e, 0x68, 0xe3, 0x1c, 0x8d, 0xa1, 0x1f, 0x27, 0x94,
+ 0x84, 0x21, 0xf3, 0x88, 0xa0, 0xea, 0x14, 0x3a, 0x4e, 0x51, 0x84, 0xdf, 0x83, 0x5b, 0x8d, 0xe0,
+ 0x3a, 0x40, 0xfc, 0xb8, 0x12, 0x3d, 0x9b, 0x4e, 0x83, 0x56, 0xae, 0xf1, 0xed, 0x5a, 0xd4, 0xca,
+ 0xd2, 0xe0, 0x3e, 0xa9, 0xec, 0x86, 0x94, 0x44, 0x69, 0xdc, 0x0a, 0xb8, 0x1a, 0x71, 0x66, 0x9a,
+ 0x23, 0x6f, 0xea, 0xe2, 0xd8, 0x67, 0x61, 0x48, 0x3d, 0x11, 0xb0, 0x28, 0x83, 0xdd, 0x02, 0xf0,
+ 0x72, 0xa1, 0x29, 0x95, 0x82, 0x04, 0xdb, 0x30, 0xaa, 0x9b, 0x1a, 0xd8, 0xdf, 0x2c, 0x58, 0xfb,
+ 0x9a, 0xf3, 0xe0, 0x34, 0xd2, 0x6e, 0x5b, 0x1d, 0x7f, 0xd9, 0xe1, 0xb5, 0xaa, 0xc3, 0xea, 0xf5,
+ 0x74, 0x6a, 0xd7, 0x23, 0x35, 0x12, 0x1a, 0x87, 0x81, 0x47, 0x14, 0x44, 0x57, 0x41, 0x14, 0x45,
+ 0x68, 0x08, 0x1d, 0x21, 0xc2, 0x51, 0x4f, 0xed, 0xc8, 0x25, 0xde, 0x80, 0xf5, 0x72, 0xa4, 0x26,
+ 0x85, 0xcf, 0x61, 0x53, 0x4b, 0x8e, 0x2e, 0x23, 0xef, 0x48, 0x75, 0x42, 0xab, 0x03, 0xff, 0xcb,
+ 0x82, 0x51, 0xdd, 0xd0, 0x54, 0xf0, 0x7f, 0xcd, 0xff, 0x9f, 0x66, 0x87, 0xde, 0x87, 0xbe, 0x20,
+ 0x41, 0xe8, 0xb2, 0x93, 0x13, 0x4e, 0xc5, 0x68, 0x71, 0x6c, 0x4d, 0xba, 0x0e, 0x48, 0xd1, 0x73,
+ 0x25, 0x41, 0xdb, 0x30, 0xf4, 0x74, 0x15, 0xbb, 0x09, 0xbd, 0x08, 0xb8, 0x44, 0x5e, 0x52, 0x81,
+ 0x5d, 0xf7, 0xb2, 0xea, 0xd6, 0x62, 0x84, 0x61, 0x35, 0xf0, 0xdf, 0xb8, 0x6a, 0x78, 0xa8, 0xd6,
+ 0x5f, 0x56, 0x68, 0xfd, 0xc0, 0x7f, 0xf3, 0x34, 0x08, 0xe9, 0x91, 0x9c, 0x00, 0xdf, 0xc1, 0x9a,
+ 0x4e, 0xfe, 0x29, 0x0b, 0x43, 0xf6, 0xba, 0xd5, 0xbd, 0xaf, 0x43, 0x8f, 0x07, 0x91, 0xa7, 0x1b,
+ 0xae, 0xeb, 0xe8, 0x0f, 0xfc, 0x04, 0xd6, 0xcb, 0x48, 0xe6, 0x08, 0xef, 0xc0, 0x40, 0x45, 0xe0,
+ 0xb1, 0x48, 0xd0, 0x48, 0x28, 0xb4, 0x81, 0xd3, 0x97, 0xb2, 0x7d, 0x2d, 0xc2, 0x9f, 0x02, 0xd2,
+ 0xa6, 0xcf, 0x58, 0x1a, 0xb5, 0xeb, 0xbf, 0x1b, 0x59, 0xdc, 0xc6, 0xc4, 0x14, 0xc1, 0xa3, 0x2c,
+ 0x88, 0x17, 0xd1, 0xb4, 0x35, 0xd6, 0x26, 0xdc, 0xa8, 0x18, 0x19, 0xb4, 0xbd, 0xcc, 0x49, 0xf9,
+ 0x31, 0x78, 0x2b, 0xd8, 0x46, 0x16, 0x41, 0xf9, 0x3d, 0xc0, 0xbf, 0x5b, 0xb0, 0xe1, 0x98, 0xab,
+ 0xa7, 0xff, 0x6f, 0x93, 0x15, 0x8b, 0xac, 0x33, 0xb3, 0xc8, 0xba, 0x57, 0x45, 0x36, 0x81, 0x21,
+ 0x67, 0x69, 0xe2, 0x51, 0xd7, 0x27, 0x82, 0xb8, 0x11, 0xf3, 0xa9, 0xa9, 0xc1, 0x77, 0xb4, 0xfc,
+ 0x80, 0x08, 0xf2, 0x3d, 0xf3, 0x29, 0xbe, 0x09, 0x9b, 0xb5, 0xa0, 0x4d, 0x42, 0x11, 0x5c, 0xdf,
+ 0x67, 0xf1, 0xa5, 0xac, 0xa4, 0x96, 0x89, 0xf4, 0x03, 0xee, 0x66, 0x05, 0xa9, 0x32, 0x59, 0x76,
+ 0x56, 0x02, 0x7e, 0xa8, 0xab, 0xd1, 0xec, 0xfb, 0x44, 0xe8, 0xfd, 0x4e, 0xb6, 0x7f, 0x40, 0x84,
+ 0xdc, 0xc7, 0x9f, 0xc1, 0xf0, 0xca, 0x5f, 0xfb, 0xda, 0xfa, 0x12, 0x6e, 0x39, 0x94, 0xf8, 0xa6,
+ 0x34, 0x65, 0xd9, 0xb7, 0x1f, 0x0d, 0x7f, 0x5a, 0x70, 0xbb, 0xd9, 0xb8, 0xcd, 0x78, 0xd8, 0x01,
+ 0x94, 0xb7, 0x9f, 0x08, 0xa6, 0x94, 0x0b, 0x32, 0x8d, 0x4d, 0xcf, 0x0c, 0x4d, 0x0f, 0xfe, 0x90,
+ 0xc9, 0xeb, 0xcd, 0xda, 0xa9, 0x35, 0xab, 0x44, 0xcc, 0xce, 0xa7, 0x80, 0xd8, 0xd5, 0x88, 0xbe,
+ 0x3e, 0xa7, 0x12, 0x62, 0xae, 0xad, 0x10, 0x7b, 0x1a, 0xd1, 0x28, 0xaa, 0xf6, 0xff, 0x11, 0xe0,
+ 0x20, 0xe0, 0xe7, 0x3a, 0x2d, 0x59, 0x29, 0x7e, 0x90, 0x98, 0xa7, 0x43, 0x2e, 0xa5, 0x84, 0x84,
+ 0xa1, 0x09, 0x5a, 0x2e, 0x25, 0x8d, 0x48, 0x39, 0xf5, 0x4d, 0x78, 0x6a, 0x2d, 0x65, 0x27, 0x09,
+ 0xa5, 0x26, 0x12, 0xb5, 0xc6, 0xbf, 0x5a, 0xb0, 0xf2, 0x8c, 0x4e, 0x0d, 0xf2, 0x16, 0xc0, 0x29,
+ 0x4b, 0x58, 0x2a, 0x82, 0x88, 0x72, 0xe5, 0xa0, 0xe7, 0x14, 0x24, 0xff, 0xde, 0x8f, 0xa2, 0x35,
+ 0x34, 0x3c, 0x31, 0xc9, 0xa9, 0xb5, 0x94, 0x9d, 0x51, 0x12, 0x9b, 0xe9, 0xa9, 0xd6, 0x6a, 0x68,
+ 0x09, 0xe2, 0x9d, 0xab, 0x61, 0x29, 0x87, 0x96, 0xfc, 0xd8, 0xfb, 0x63, 0x00, 0x03, 0x33, 0xfc,
+ 0x15, 0x2b, 0x43, 0x3f, 0x43, 0xbf, 0xc0, 0xe6, 0xd0, 0x87, 0x75, 0xd2, 0x56, 0x67, 0x87, 0xf6,
+ 0xdd, 0x39, 0x5a, 0xa6, 0x63, 0x16, 0x50, 0x04, 0xef, 0xd6, 0xd8, 0x12, 0xba, 0x5f, 0xb7, 0x9e,
+ 0xc5, 0xc5, 0xec, 0x07, 0xad, 0x74, 0x73, 0x7f, 0x02, 0xd6, 0x1a, 0xe8, 0x0f, 0xda, 0x99, 0x83,
+ 0x52, 0xa2, 0x60, 0xf6, 0xc3, 0x96, 0xda, 0xb9, 0xd7, 0x57, 0x80, 0xea, 0xdc, 0x08, 0x3d, 0x98,
+ 0x0b, 0x73, 0xc5, 0xbd, 0xec, 0x9d, 0x76, 0xca, 0x33, 0x13, 0xd5, 0xac, 0x69, 0x6e, 0xa2, 0x25,
+ 0x5e, 0x36, 0x37, 0xd1, 0x0a, 0x15, 0x5b, 0x40, 0xe7, 0x30, 0xac, 0x32, 0x2a, 0xb4, 0x3d, 0x8b,
+ 0xe6, 0xd7, 0x08, 0x9b, 0x7d, 0xbf, 0x8d, 0x6a, 0xee, 0xcc, 0x85, 0x41, 0x91, 0xf7, 0xa0, 0x86,
+ 0xa2, 0x6b, 0x60, 0x70, 0xf6, 0xbd, 0x79, 0x6a, 0xc5, 0x6c, 0xaa, 0x3c, 0xa8, 0x29, 0x9b, 0x19,
+ 0x24, 0xab, 0x29, 0x9b, 0x59, 0xb4, 0x0a, 0x2f, 0x20, 0x92, 0xf5, 0x9d, 0x66, 0x0b, 0x4d, 0xd9,
+ 0x34, 0xf0, 0x92, 0xa6, 0x6c, 0x9a, 0x48, 0x07, 0x5e, 0xf8, 0xc4, 0x92, 0xad, 0x5c, 0xa0, 0x08,
+ 0x4d, 0xad, 0x5c, 0x27, 0x1d, 0xf6, 0xdd, 0x39, 0x5a, 0x79, 0x02, 0xc7, 0xb0, 0x5a, 0x22, 0x0d,
+ 0x68, 0x66, 0x68, 0x65, 0x2a, 0x62, 0x7f, 0x34, 0x57, 0xaf, 0x78, 0xe5, 0x45, 0x2e, 0x31, 0xfb,
+ 0x90, 0xca, 0xe3, 0xe8, 0xde, 0x3c, 0xb5, 0xdc, 0xc1, 0x19, 0x5c, 0xaf, 0x3c, 0xef, 0x68, 0x52,
+ 0x37, 0x6e, 0xa6, 0x2d, 0xf6, 0x76, 0x0b, 0xcd, 0xdc, 0xd3, 0x6b, 0x58, 0x6f, 0x7a, 0x49, 0xd1,
+ 0xc3, 0x26, 0x90, 0x99, 0xcf, 0xb5, 0xbd, 0xdb, 0x56, 0x3d, 0x77, 0xfc, 0x02, 0x96, 0x33, 0xda,
+ 0x80, 0xee, 0xd4, 0xad, 0x2b, 0x14, 0xc6, 0xc6, 0x6f, 0x53, 0xb9, 0x2a, 0xae, 0xe3, 0x45, 0xf5,
+ 0xff, 0xe0, 0xd1, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x13, 0x86, 0xf1, 0x0f, 0x56, 0x10, 0x00,
+ 0x00,
}
diff --git a/weed/server/volume_grpc_follow.go b/weed/server/volume_grpc_follow.go
new file mode 100644
index 000000000..cc5dcc78e
--- /dev/null
+++ b/weed/server/volume_grpc_follow.go
@@ -0,0 +1,67 @@
+package weed_server
+
+import (
+ "context"
+ "fmt"
+ "github.com/chrislusf/seaweedfs/weed/storage/types"
+ "io"
+ "os"
+
+ "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
+ "github.com/chrislusf/seaweedfs/weed/storage"
+)
+
+func (vs *VolumeServer) VolumeFollow(req *volume_server_pb.VolumeFollowRequest, stream volume_server_pb.VolumeServer_VolumeFollowServer) error {
+
+ v := vs.store.GetVolume(storage.VolumeId(req.VolumeId))
+ if v == nil {
+ return fmt.Errorf("not found volume id %d", req.VolumeId)
+ }
+
+ stopOffset := v.Size()
+ foundOffset, isLastOne, err := v.BinarySearchByAppendAtNs(req.Since)
+ if err != nil {
+ return fmt.Errorf("fail to locate by appendAtNs %d: %s", req.Since, err)
+ }
+
+ if isLastOne {
+ return nil
+ }
+
+ startOffset := int64(foundOffset) * int64(types.NeedlePaddingSize)
+
+ buf := make([]byte, 1024*1024*2)
+ return sendFileContent(v.DataFile(), buf, startOffset, stopOffset, stream)
+
+}
+
+func (vs *VolumeServer) VolumeSyncStatus(ctx context.Context, req *volume_server_pb.VolumeSyncStatusRequest) (*volume_server_pb.VolumeSyncStatusResponse, error) {
+
+ v := vs.store.GetVolume(storage.VolumeId(req.VolumeId))
+ if v == nil {
+ return nil, fmt.Errorf("not found volume id %d", req.VolumeId)
+ }
+
+ resp := v.GetVolumeSyncStatus()
+
+ return resp, nil
+
+}
+
+func sendFileContent(datFile *os.File, buf []byte, startOffset, stopOffset int64, stream volume_server_pb.VolumeServer_VolumeFollowServer) error {
+ var blockSizeLimit = int64(len(buf))
+ for i := int64(0); i < stopOffset-startOffset; i += blockSizeLimit {
+ n, readErr := datFile.ReadAt(buf, startOffset+i)
+ if readErr == nil || readErr == io.EOF {
+ resp := &volume_server_pb.VolumeFollowResponse{}
+ resp.FileContent = buf[:int64(n)]
+ sendErr := stream.Send(resp)
+ if sendErr != nil {
+ return sendErr
+ }
+ } else {
+ return readErr
+ }
+ }
+ return nil
+}
diff --git a/weed/server/volume_grpc_replicate.go b/weed/server/volume_grpc_replicate.go
index 20a85fd6f..1a31a37f3 100644
--- a/weed/server/volume_grpc_replicate.go
+++ b/weed/server/volume_grpc_replicate.go
@@ -113,7 +113,7 @@ func (vs *VolumeServer) ReadVolumeFileStatus(ctx context.Context, req *volume_se
return resp, nil
}
-func (vs *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream volume_server_pb.VolumeServer_CopyFileServer) (error) {
+func (vs *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream volume_server_pb.VolumeServer_CopyFileServer) error {
v := vs.store.GetVolume(storage.VolumeId(req.VolumeId))
if v == nil {
diff --git a/weed/server/volume_grpc_sync.go b/weed/server/volume_grpc_sync.go
deleted file mode 100644
index 971258689..000000000
--- a/weed/server/volume_grpc_sync.go
+++ /dev/null
@@ -1,101 +0,0 @@
-package weed_server
-
-import (
- "context"
- "fmt"
-
- "github.com/chrislusf/seaweedfs/weed/glog"
- "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
- "github.com/chrislusf/seaweedfs/weed/storage"
- "github.com/chrislusf/seaweedfs/weed/storage/types"
-)
-
-func (vs *VolumeServer) VolumeSyncStatus(ctx context.Context, req *volume_server_pb.VolumeSyncStatusRequest) (*volume_server_pb.VolumeSyncStatusResponse, error) {
-
- v := vs.store.GetVolume(storage.VolumeId(req.VolumeId))
- if v == nil {
- return nil, fmt.Errorf("not found volume id %d", req.VolumeId)
- }
-
- resp := v.GetVolumeSyncStatus()
-
- glog.V(2).Infof("volume sync status %d", req.VolumeId)
-
- return resp, nil
-
-}
-
-func (vs *VolumeServer) VolumeSyncIndex(req *volume_server_pb.VolumeSyncIndexRequest, stream volume_server_pb.VolumeServer_VolumeSyncIndexServer) error {
-
- v := vs.store.GetVolume(storage.VolumeId(req.VolumeId))
- if v == nil {
- return fmt.Errorf("not found volume id %d", req.VolumeId)
- }
-
- content, err := v.IndexFileContent()
-
- if err != nil {
- glog.Errorf("sync volume %d index: %v", req.VolumeId, err)
- } else {
- glog.V(2).Infof("sync volume %d index", req.VolumeId)
- }
-
- const blockSizeLimit = 1024 * 1024 * 2
- for i := 0; i < len(content); i += blockSizeLimit {
- blockSize := len(content) - i
- if blockSize > blockSizeLimit {
- blockSize = blockSizeLimit
- }
- resp := &volume_server_pb.VolumeSyncIndexResponse{}
- resp.IndexFileContent = content[i : i+blockSize]
- stream.Send(resp)
- }
-
- return nil
-
-}
-
-func (vs *VolumeServer) VolumeSyncData(req *volume_server_pb.VolumeSyncDataRequest, stream volume_server_pb.VolumeServer_VolumeSyncDataServer) error {
-
- v := vs.store.GetVolume(storage.VolumeId(req.VolumeId))
- if v == nil {
- return fmt.Errorf("not found volume id %d", req.VolumeId)
- }
-
- if uint32(v.SuperBlock.CompactRevision) != req.Revision {
- return fmt.Errorf("requested volume revision is %d, but current revision is %d", req.Revision, v.SuperBlock.CompactRevision)
- }
-
- content, err := storage.ReadNeedleBlob(v.DataFile(), int64(req.Offset)*types.NeedlePaddingSize, req.Size, v.Version())
- if err != nil {
- return fmt.Errorf("read offset:%d size:%d", req.Offset, req.Size)
- }
-
- id, err := types.ParseNeedleId(req.NeedleId)
- if err != nil {
- return fmt.Errorf("parsing needle id %s: %v", req.NeedleId, err)
- }
- n := new(storage.Needle)
- n.ParseNeedleHeader(content)
- if id != n.Id {
- return fmt.Errorf("expected file entry id %d, but found %d", id, n.Id)
- }
-
- if err != nil {
- glog.Errorf("sync volume %d data: %v", req.VolumeId, err)
- }
-
- const blockSizeLimit = 1024 * 1024 * 2
- for i := 0; i < len(content); i += blockSizeLimit {
- blockSize := len(content) - i
- if blockSize > blockSizeLimit {
- blockSize = blockSizeLimit
- }
- resp := &volume_server_pb.VolumeSyncDataResponse{}
- resp.FileContent = content[i : i+blockSize]
- stream.Send(resp)
- }
-
- return nil
-
-}
diff --git a/weed/shell/command_fs_du.go b/weed/shell/command_fs_du.go
index 1206596b0..3fecac9a8 100644
--- a/weed/shell/command_fs_du.go
+++ b/weed/shell/command_fs_du.go
@@ -42,10 +42,10 @@ func (c *commandFsDu) Do(args []string, commandEnv *commandEnv, writer io.Writer
dir, name := filer2.FullPath(path).DirAndName()
if strings.HasSuffix(path, "/") {
- if path == "/"{
+ if path == "/" {
dir, name = "/", ""
- }else{
- dir, name = path[0 : len(path)-1], ""
+ } else {
+ dir, name = path[0:len(path)-1], ""
}
}
diff --git a/weed/storage/store_vacuum.go b/weed/storage/store_vacuum.go
index cc0521491..0036315c8 100644
--- a/weed/storage/store_vacuum.go
+++ b/weed/storage/store_vacuum.go
@@ -20,7 +20,7 @@ func (s *Store) CompactVolume(vid VolumeId, preallocate int64) error {
}
func (s *Store) CommitCompactVolume(vid VolumeId) error {
if v := s.findVolume(vid); v != nil {
- return v.commitCompact()
+ return v.CommitCompact()
}
return fmt.Errorf("volume id %d is not found during commit compact", vid)
}
diff --git a/weed/storage/volume_follow.go b/weed/storage/volume_follow.go
new file mode 100644
index 000000000..b8353f9d1
--- /dev/null
+++ b/weed/storage/volume_follow.go
@@ -0,0 +1,253 @@
+package storage
+
+import (
+ "context"
+ "fmt"
+ "github.com/chrislusf/seaweedfs/weed/operation"
+ "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
+ . "github.com/chrislusf/seaweedfs/weed/storage/types"
+ "google.golang.org/grpc"
+ "io"
+ "os"
+)
+
+func (v *Volume) GetVolumeSyncStatus() *volume_server_pb.VolumeSyncStatusResponse {
+ var syncStatus = &volume_server_pb.VolumeSyncStatusResponse{}
+ if stat, err := v.dataFile.Stat(); err == nil {
+ syncStatus.TailOffset = uint64(stat.Size())
+ }
+ syncStatus.Collection = v.Collection
+ syncStatus.IdxFileSize = v.nm.IndexFileSize()
+ syncStatus.CompactRevision = uint32(v.SuperBlock.CompactRevision)
+ syncStatus.Ttl = v.SuperBlock.Ttl.String()
+ syncStatus.Replication = v.SuperBlock.ReplicaPlacement.String()
+ return syncStatus
+}
+
+// The volume sync with a master volume via 2 steps:
+// 1. The slave checks master side to find subscription checkpoint
+// to setup the replication.
+// 2. The slave receives the updates from master
+
+/*
+Assume the slave volume needs to follow the master volume.
+
+The master volume could be compacted, and could be many files ahead of
+slave volume.
+
+Step 0: // implemented in command/backup.go, to avoid dat file size overflow.
+0.1 If slave compact version is less than the master, do a local compaction, and set
+local compact version the same as the master.
+0.2 If the slave size is still bigger than the master, discard local copy and do a full copy.
+
+Step 1:
+The slave volume ask the master by the last modification time t.
+The master do a binary search in volume (use .idx as an array, and check the appendAtNs in .dat file),
+to find the first entry with appendAtNs > t.
+
+Step 2:
+The master send content bytes to the slave. The bytes are not chunked by needle.
+
+Step 3:
+The slave generate the needle map for the new bytes. (This may be optimized to incrementally
+update needle map when receiving new .dat bytes. But seems not necessary now.)
+
+*/
+
+func (v *Volume) Follow(volumeServer string, grpcDialOption grpc.DialOption) error {
+
+ ctx := context.Background()
+
+ startFromOffset := v.Size()
+ appendAtNs, err := v.findLastAppendAtNs()
+ if err != nil {
+ return err
+ }
+
+ err = operation.WithVolumeServerClient(volumeServer, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+
+ stream, err := client.VolumeFollow(ctx, &volume_server_pb.VolumeFollowRequest{
+ VolumeId: uint32(v.Id),
+ Since: appendAtNs,
+ })
+ if err != nil {
+ return err
+ }
+
+ v.dataFile.Seek(startFromOffset, io.SeekStart)
+
+ for {
+ resp, recvErr := stream.Recv()
+ if recvErr != nil {
+ if recvErr == io.EOF {
+ break
+ } else {
+ return recvErr
+ }
+ }
+
+ _, writeErr := v.dataFile.Write(resp.FileContent)
+ if writeErr != nil {
+ return writeErr
+ }
+ }
+
+ return nil
+
+ })
+
+ if err != nil {
+ return err
+ }
+
+ // add to needle map
+ return ScanVolumeFileFrom(v.version, v.dataFile, startFromOffset, &VolumeFileScanner4GenIdx{v: v})
+
+}
+
+func (v *Volume) findLastAppendAtNs() (uint64, error) {
+ offset, err := v.locateLastAppendEntry()
+ if err != nil {
+ return 0, err
+ }
+ if offset == 0 {
+ return 0, nil
+ }
+ return v.readAppendAtNs(offset)
+}
+
+func (v *Volume) locateLastAppendEntry() (Offset, error) {
+ indexFile, e := os.OpenFile(v.FileName()+".idx", os.O_RDONLY, 0644)
+ if e != nil {
+ return 0, fmt.Errorf("cannot read %s.idx: %v", v.FileName(), e)
+ }
+ defer indexFile.Close()
+
+ fi, err := indexFile.Stat()
+ if err != nil {
+ return 0, fmt.Errorf("file %s stat error: %v", indexFile.Name(), err)
+ }
+ fileSize := fi.Size()
+ if fileSize%NeedleEntrySize != 0 {
+ return 0, fmt.Errorf("unexpected file %s size: %d", indexFile.Name(), fileSize)
+ }
+ if fileSize == 0 {
+ return 0, nil
+ }
+
+ bytes := make([]byte, NeedleEntrySize)
+ n, e := indexFile.ReadAt(bytes, fileSize-NeedleEntrySize)
+ if n != NeedleEntrySize {
+ return 0, fmt.Errorf("file %s read error: %v", indexFile.Name(), e)
+ }
+ _, offset, _ := IdxFileEntry(bytes)
+
+ return offset, nil
+}
+
+func (v *Volume) readAppendAtNs(offset Offset) (uint64, error) {
+
+ n, bodyLength, err := ReadNeedleHeader(v.dataFile, v.SuperBlock.version, int64(offset)*NeedlePaddingSize)
+ if err != nil {
+ return 0, fmt.Errorf("ReadNeedleHeader: %v", err)
+ }
+ err = n.ReadNeedleBody(v.dataFile, v.SuperBlock.version, int64(offset)*NeedlePaddingSize+int64(NeedleEntrySize), bodyLength)
+ if err != nil {
+ return 0, fmt.Errorf("ReadNeedleBody offset %d, bodyLength %d: %v", int64(offset)*NeedlePaddingSize, bodyLength, err)
+ }
+ return n.AppendAtNs, nil
+
+}
+
+// on server side
+func (v *Volume) BinarySearchByAppendAtNs(sinceNs uint64) (offset Offset, isLast bool, err error) {
+ indexFile, openErr := os.OpenFile(v.FileName()+".idx", os.O_RDONLY, 0644)
+ if openErr != nil {
+ err = fmt.Errorf("cannot read %s.idx: %v", v.FileName(), openErr)
+ return
+ }
+ defer indexFile.Close()
+
+ fi, statErr := indexFile.Stat()
+ if statErr != nil {
+ err = fmt.Errorf("file %s stat error: %v", indexFile.Name(), statErr)
+ return
+ }
+ fileSize := fi.Size()
+ if fileSize%NeedleEntrySize != 0 {
+ err = fmt.Errorf("unexpected file %s size: %d", indexFile.Name(), fileSize)
+ return
+ }
+
+ bytes := make([]byte, NeedleEntrySize)
+ entryCount := fileSize / NeedleEntrySize
+ l := int64(0)
+ h := entryCount
+
+ for l < h {
+
+ m := (l + h) / 2
+
+ if m == entryCount {
+ return 0, true, nil
+ }
+
+ // read the appendAtNs for entry m
+ offset, err = v.readAppendAtNsForIndexEntry(indexFile, bytes, m)
+ if err != nil {
+ return
+ }
+
+ mNs, nsReadErr := v.readAppendAtNs(offset)
+ if nsReadErr != nil {
+ err = nsReadErr
+ return
+ }
+
+ // move the boundary
+ if mNs <= sinceNs {
+ l = m + 1
+ } else {
+ h = m
+ }
+
+ }
+
+ if l == entryCount {
+ return 0, true, nil
+ }
+
+ offset, err = v.readAppendAtNsForIndexEntry(indexFile, bytes, l)
+
+ return offset, false, err
+
+}
+
+// bytes is of size NeedleEntrySize
+func (v *Volume) readAppendAtNsForIndexEntry(indexFile *os.File, bytes []byte, m int64) (Offset, error) {
+ if _, readErr := indexFile.ReadAt(bytes, m*NeedleEntrySize); readErr != nil && readErr != io.EOF {
+ return 0, readErr
+ }
+ _, offset, _ := IdxFileEntry(bytes)
+ return offset, nil
+}
+
+// generate the volume idx
+type VolumeFileScanner4GenIdx struct {
+ v *Volume
+}
+
+func (scanner *VolumeFileScanner4GenIdx) VisitSuperBlock(superBlock SuperBlock) error {
+ return nil
+
+}
+func (scanner *VolumeFileScanner4GenIdx) ReadNeedleBody() bool {
+ return false
+}
+
+func (scanner *VolumeFileScanner4GenIdx) VisitNeedle(n *Needle, offset int64) error {
+ if n.Size > 0 && n.Size != TombstoneFileSize {
+ return scanner.v.nm.Put(n.Id, Offset(offset/NeedlePaddingSize), n.Size)
+ }
+ return scanner.v.nm.Delete(n.Id, Offset(offset/NeedlePaddingSize))
+}
diff --git a/weed/storage/volume_follow_test.go b/weed/storage/volume_follow_test.go
new file mode 100644
index 000000000..3291d203a
--- /dev/null
+++ b/weed/storage/volume_follow_test.go
@@ -0,0 +1,39 @@
+package storage
+
+import "testing"
+
+func TestBinarySearch(t *testing.T) {
+ var testInput []int
+ testInput = []int{-1, 0, 3, 5, 9, 12}
+
+ if 3 != binarySearchForLargerThanTarget(testInput, 4) {
+ t.Errorf("failed to find target %d", 4)
+ }
+ if 3 != binarySearchForLargerThanTarget(testInput, 3) {
+ t.Errorf("failed to find target %d", 3)
+ }
+ if 6 != binarySearchForLargerThanTarget(testInput, 12) {
+ t.Errorf("failed to find target %d", 12)
+ }
+ if 1 != binarySearchForLargerThanTarget(testInput, -1) {
+ t.Errorf("failed to find target %d", -1)
+ }
+ if 0 != binarySearchForLargerThanTarget(testInput, -2) {
+ t.Errorf("failed to find target %d", -2)
+ }
+
+}
+
+func binarySearchForLargerThanTarget(nums []int, target int) int {
+ l := 0
+ h := len(nums)
+ for l < h {
+ m := (l + h) / 2
+ if nums[m] <= target {
+ l = m + 1
+ } else {
+ h = m
+ }
+ }
+ return l
+}
diff --git a/weed/storage/volume_read_write.go b/weed/storage/volume_read_write.go
index ed9729c84..5366a547d 100644
--- a/weed/storage/volume_read_write.go
+++ b/weed/storage/volume_read_write.go
@@ -180,30 +180,37 @@ func ScanVolumeFile(dirname string, collection string, id VolumeId,
volumeFileScanner VolumeFileScanner) (err error) {
var v *Volume
if v, err = loadVolumeWithoutIndex(dirname, collection, id, needleMapKind); err != nil {
- return fmt.Errorf("Failed to load volume %d: %v", id, err)
+ return fmt.Errorf("failed to load volume %d: %v", id, err)
}
if err = volumeFileScanner.VisitSuperBlock(v.SuperBlock); err != nil {
- return fmt.Errorf("Failed to process volume %d super block: %v", id, err)
+ return fmt.Errorf("failed to process volume %d super block: %v", id, err)
}
defer v.Close()
version := v.Version()
offset := int64(v.SuperBlock.BlockSize())
- n, rest, e := ReadNeedleHeader(v.dataFile, version, offset)
+
+ return ScanVolumeFileFrom(version, v.dataFile, offset, volumeFileScanner)
+}
+
+func ScanVolumeFileFrom(version Version, dataFile *os.File, offset int64, volumeFileScanner VolumeFileScanner) (err error) {
+ n, rest, e := ReadNeedleHeader(dataFile, version, offset)
if e != nil {
- err = fmt.Errorf("cannot read needle header: %v", e)
- return
+ if e == io.EOF {
+ return nil
+ }
+ return fmt.Errorf("cannot read %s at offset %d: %v", dataFile.Name(), offset, e)
}
for n != nil {
if volumeFileScanner.ReadNeedleBody() {
- if err = n.ReadNeedleBody(v.dataFile, version, offset+NeedleEntrySize, rest); err != nil {
+ if err = n.ReadNeedleBody(dataFile, version, offset+NeedleEntrySize, rest); err != nil {
glog.V(0).Infof("cannot read needle body: %v", err)
//err = fmt.Errorf("cannot read needle body: %v", err)
//return
}
}
- err = volumeFileScanner.VisitNeedle(n, offset)
+ err := volumeFileScanner.VisitNeedle(n, offset)
if err == io.EOF {
return nil
}
@@ -212,14 +219,13 @@ func ScanVolumeFile(dirname string, collection string, id VolumeId,
}
offset += NeedleEntrySize + rest
glog.V(4).Infof("==> new entry offset %d", offset)
- if n, rest, err = ReadNeedleHeader(v.dataFile, version, offset); err != nil {
+ if n, rest, err = ReadNeedleHeader(dataFile, version, offset); err != nil {
if err == io.EOF {
return nil
}
- return fmt.Errorf("cannot read needle header: %v", err)
+ return fmt.Errorf("cannot read needle header at offset %d: %v", offset, err)
}
glog.V(4).Infof("new entry needle size:%d rest:%d", n.Size, rest)
}
-
- return
+ return nil
}
diff --git a/weed/storage/volume_sync.go b/weed/storage/volume_sync.go
deleted file mode 100644
index 827e6685a..000000000
--- a/weed/storage/volume_sync.go
+++ /dev/null
@@ -1,225 +0,0 @@
-package storage
-
-import (
- "context"
- "fmt"
- "google.golang.org/grpc"
- "io"
- "os"
- "sort"
-
- "github.com/chrislusf/seaweedfs/weed/glog"
- "github.com/chrislusf/seaweedfs/weed/operation"
- "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
- "github.com/chrislusf/seaweedfs/weed/storage/needle"
- . "github.com/chrislusf/seaweedfs/weed/storage/types"
-)
-
-// The volume sync with a master volume via 2 steps:
-// 1. The slave checks master side to find subscription checkpoint
-// to setup the replication.
-// 2. The slave receives the updates from master
-
-/*
-Assume the slave volume needs to follow the master volume.
-
-The master volume could be compacted, and could be many files ahead of
-slave volume.
-
-Step 1:
-The slave volume will ask the master volume for a snapshot
-of (existing file entries, last offset, number of compacted times).
-
-For each entry x in master existing file entries:
- if x does not exist locally:
- add x locally
-
-For each entry y in local slave existing file entries:
- if y does not exist on master:
- delete y locally
-
-Step 2:
-After this, use the last offset and number of compacted times to request
-the master volume to send a new file, and keep looping. If the number of
-compacted times is changed, go back to step 1 (very likely this can be
-optimized more later).
-
-*/
-
-func (v *Volume) Synchronize(volumeServer string, grpcDialOption grpc.DialOption) (err error) {
- var lastCompactRevision uint16 = 0
- var compactRevision uint16 = 0
- var masterMap *needle.CompactMap
- for i := 0; i < 3; i++ {
- if masterMap, _, compactRevision, err = fetchVolumeFileEntries(volumeServer, grpcDialOption, v.Id); err != nil {
- return fmt.Errorf("Failed to sync volume %d entries with %s: %v", v.Id, volumeServer, err)
- }
- if lastCompactRevision != compactRevision && lastCompactRevision != 0 {
- if err = v.Compact(0); err != nil {
- return fmt.Errorf("Compact Volume before synchronizing %v", err)
- }
- if err = v.commitCompact(); err != nil {
- return fmt.Errorf("Commit Compact before synchronizing %v", err)
- }
- }
- lastCompactRevision = compactRevision
- if err = v.trySynchronizing(volumeServer, grpcDialOption, masterMap, compactRevision); err == nil {
- return
- }
- }
- return
-}
-
-type ByOffset []needle.NeedleValue
-
-func (a ByOffset) Len() int { return len(a) }
-func (a ByOffset) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
-func (a ByOffset) Less(i, j int) bool { return a[i].Offset < a[j].Offset }
-
-// trySynchronizing sync with remote volume server incrementally by
-// make up the local and remote delta.
-func (v *Volume) trySynchronizing(volumeServer string, grpcDialOption grpc.DialOption, masterMap *needle.CompactMap, compactRevision uint16) error {
- slaveIdxFile, err := os.Open(v.nm.IndexFileName())
- if err != nil {
- return fmt.Errorf("Open volume %d index file: %v", v.Id, err)
- }
- defer slaveIdxFile.Close()
- slaveMap, err := LoadBtreeNeedleMap(slaveIdxFile)
- if err != nil {
- return fmt.Errorf("Load volume %d index file: %v", v.Id, err)
- }
- var delta []needle.NeedleValue
- if err := masterMap.Visit(func(needleValue needle.NeedleValue) error {
- if needleValue.Key == NeedleIdEmpty {
- return nil
- }
- if _, ok := slaveMap.Get(needleValue.Key); ok {
- return nil // skip intersection
- }
- delta = append(delta, needleValue)
- return nil
- }); err != nil {
- return fmt.Errorf("Add master entry: %v", err)
- }
- if err := slaveMap.m.Visit(func(needleValue needle.NeedleValue) error {
- if needleValue.Key == NeedleIdEmpty {
- return nil
- }
- if _, ok := masterMap.Get(needleValue.Key); ok {
- return nil // skip intersection
- }
- needleValue.Size = 0
- delta = append(delta, needleValue)
- return nil
- }); err != nil {
- return fmt.Errorf("Remove local entry: %v", err)
- }
-
- // simulate to same ordering of remote .dat file needle entries
- sort.Sort(ByOffset(delta))
-
- // make up the delta
- fetchCount := 0
- for _, needleValue := range delta {
- if needleValue.Size == 0 {
- // remove file entry from local
- v.removeNeedle(needleValue.Key)
- continue
- }
- // add master file entry to local data file
- if err := v.fetchNeedle(volumeServer, grpcDialOption, needleValue, compactRevision); err != nil {
- glog.V(0).Infof("Fetch needle %v from %s: %v", needleValue, volumeServer, err)
- return err
- }
- fetchCount++
- }
- glog.V(1).Infof("Fetched %d needles from %s", fetchCount, volumeServer)
- return nil
-}
-
-func fetchVolumeFileEntries(volumeServer string, grpcDialOption grpc.DialOption, vid VolumeId) (m *needle.CompactMap, lastOffset uint64, compactRevision uint16, err error) {
- m = needle.NewCompactMap()
-
- syncStatus, err := operation.GetVolumeSyncStatus(volumeServer, grpcDialOption, uint32(vid))
- if err != nil {
- return m, 0, 0, err
- }
-
- total := 0
- err = operation.GetVolumeIdxEntries(volumeServer, grpcDialOption, uint32(vid), func(key NeedleId, offset Offset, size uint32) {
- // println("remote key", key, "offset", offset*NeedlePaddingSize, "size", size)
- if offset > 0 && size != TombstoneFileSize {
- m.Set(NeedleId(key), offset, size)
- } else {
- m.Delete(NeedleId(key))
- }
- total++
- })
-
- glog.V(2).Infof("server %s volume %d, entries %d, last offset %d, revision %d", volumeServer, vid, total, syncStatus.TailOffset, syncStatus.CompactRevision)
- return m, syncStatus.TailOffset, uint16(syncStatus.CompactRevision), err
-
-}
-
-func (v *Volume) GetVolumeSyncStatus() *volume_server_pb.VolumeSyncStatusResponse {
- var syncStatus = &volume_server_pb.VolumeSyncStatusResponse{}
- if stat, err := v.dataFile.Stat(); err == nil {
- syncStatus.TailOffset = uint64(stat.Size())
- }
- syncStatus.Collection = v.Collection
- syncStatus.IdxFileSize = v.nm.IndexFileSize()
- syncStatus.CompactRevision = uint32(v.SuperBlock.CompactRevision)
- syncStatus.Ttl = v.SuperBlock.Ttl.String()
- syncStatus.Replication = v.SuperBlock.ReplicaPlacement.String()
- return syncStatus
-}
-
-func (v *Volume) IndexFileContent() ([]byte, error) {
- return v.nm.IndexFileContent()
-}
-
-// removeNeedle removes one needle by needle key
-func (v *Volume) removeNeedle(key NeedleId) {
- n := new(Needle)
- n.Id = key
- v.deleteNeedle(n)
-}
-
-// fetchNeedle fetches a remote volume needle by vid, id, offset
-// The compact revision is checked first in case the remote volume
-// is compacted and the offset is invalid any more.
-func (v *Volume) fetchNeedle(volumeServer string, grpcDialOption grpc.DialOption, needleValue needle.NeedleValue, compactRevision uint16) error {
-
- return operation.WithVolumeServerClient(volumeServer, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
- stream, err := client.VolumeSyncData(context.Background(), &volume_server_pb.VolumeSyncDataRequest{
- VolumeId: uint32(v.Id),
- Revision: uint32(compactRevision),
- Offset: uint32(needleValue.Offset),
- Size: uint32(needleValue.Size),
- NeedleId: needleValue.Key.String(),
- })
- if err != nil {
- return err
- }
- var fileContent []byte
- for {
- resp, err := stream.Recv()
- if err == io.EOF {
- break
- }
- if err != nil {
- return fmt.Errorf("read needle %v: %v", needleValue.Key.String(), err)
- }
- fileContent = append(fileContent, resp.FileContent...)
- }
-
- offset, err := v.AppendBlob(fileContent)
- if err != nil {
- return fmt.Errorf("Appending volume %d error: %v", v.Id, err)
- }
- // println("add key", needleValue.Key, "offset", offset, "size", needleValue.Size)
- v.nm.Put(needleValue.Key, Offset(offset/NeedlePaddingSize), needleValue.Size)
- return nil
- })
-
-}
diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go
index b29e15843..b575277cd 100644
--- a/weed/storage/volume_vacuum.go
+++ b/weed/storage/volume_vacuum.go
@@ -38,7 +38,7 @@ func (v *Volume) Compact2() error {
return v.copyDataBasedOnIndexFile(filePath+".cpd", filePath+".cpx")
}
-func (v *Volume) commitCompact() error {
+func (v *Volume) CommitCompact() error {
glog.V(0).Infof("Committing volume %d vacuuming...", v.Id)
v.dataFileAccessLock.Lock()
defer v.dataFileAccessLock.Unlock()
@@ -53,7 +53,7 @@ func (v *Volume) commitCompact() error {
var e error
if e = v.makeupDiff(v.FileName()+".cpd", v.FileName()+".cpx", v.FileName()+".dat", v.FileName()+".idx"); e != nil {
- glog.V(0).Infof("makeupDiff in commitCompact volume %d failed %v", v.Id, e)
+ glog.V(0).Infof("makeupDiff in CommitCompact volume %d failed %v", v.Id, e)
e = os.Remove(v.FileName() + ".cpd")
if e != nil {
return e
@@ -261,7 +261,7 @@ func (scanner *VolumeFileScanner4Vacuum) VisitNeedle(n *Needle, offset int64) er
}
nv, ok := scanner.v.nm.Get(n.Id)
glog.V(4).Infoln("needle expected offset ", offset, "ok", ok, "nv", nv)
- if ok && int64(nv.Offset)*NeedlePaddingSize == offset && nv.Size > 0 {
+ if ok && int64(nv.Offset)*NeedlePaddingSize == offset && nv.Size > 0 && nv.Size != TombstoneFileSize {
if err := scanner.nm.Put(n.Id, Offset(scanner.newOffset/NeedlePaddingSize), n.Size); err != nil {
return fmt.Errorf("cannot put needle: %s", err)
}
diff --git a/weed/storage/volume_vacuum_test.go b/weed/storage/volume_vacuum_test.go
index 0bc24037d..4909885b9 100644
--- a/weed/storage/volume_vacuum_test.go
+++ b/weed/storage/volume_vacuum_test.go
@@ -85,7 +85,7 @@ func TestCompaction(t *testing.T) {
doSomeWritesDeletes(i+beforeCommitFileCount, v, t, infos)
}
- v.commitCompact()
+ v.CommitCompact()
v.Close()
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-22 06:54:34 +0000