aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--weed/pb/volume_server.proto20
-rw-r--r--weed/pb/volume_server_pb/volume_server.pb.go368
-rw-r--r--weed/server/volume_grpc_copy.go5
-rw-r--r--weed/server/volume_grpc_erasure_coding.go59
-rw-r--r--weed/server/volume_grpc_tail.go5
-rw-r--r--weed/shell/command_ec_encode.go6
-rw-r--r--weed/storage/erasure_coding/ec_locate.go12
-rw-r--r--weed/storage/erasure_coding/ec_shard.go6
-rw-r--r--weed/storage/erasure_coding/ec_test.go19
-rw-r--r--weed/storage/erasure_coding/ec_volume.go13
-rw-r--r--weed/storage/needle/needle_read_write.go16
-rw-r--r--weed/storage/store_ec.go57
12 files changed, 418 insertions, 168 deletions
diff --git a/weed/pb/volume_server.proto b/weed/pb/volume_server.proto
index 0f21c8ba5..e8d4ae61e 100644
--- a/weed/pb/volume_server.proto
+++ b/weed/pb/volume_server.proto
@@ -58,6 +58,8 @@ service VolumeServer {
}
rpc VolumeEcShardsUnmount (VolumeEcShardsUnmountRequest) returns (VolumeEcShardsUnmountResponse) {
}
+ rpc VolumeEcShardRead (VolumeEcShardReadRequest) returns (stream VolumeEcShardReadResponse) {
+ }
}
@@ -213,7 +215,7 @@ message VolumeEcShardsGenerateResponse {
message VolumeEcShardsCopyRequest {
uint32 volume_id = 1;
string collection = 2;
- repeated uint32 ec_indexes = 3;
+ repeated uint32 shard_ids = 3;
string source_data_node = 5;
}
message VolumeEcShardsCopyResponse {
@@ -222,7 +224,7 @@ message VolumeEcShardsCopyResponse {
message VolumeEcShardsDeleteRequest {
uint32 volume_id = 1;
bool should_delete_ecx = 2;
- repeated uint32 ec_indexes = 3;
+ repeated uint32 shard_ids = 3;
}
message VolumeEcShardsDeleteResponse {
}
@@ -230,18 +232,28 @@ message VolumeEcShardsDeleteResponse {
message VolumeEcShardsMountRequest {
uint32 volume_id = 1;
string collection = 2;
- repeated uint32 ec_indexes = 3;
+ repeated uint32 shard_ids = 3;
}
message VolumeEcShardsMountResponse {
}
message VolumeEcShardsUnmountRequest {
uint32 volume_id = 1;
- repeated uint32 ec_indexes = 3;
+ repeated uint32 shard_ids = 3;
}
message VolumeEcShardsUnmountResponse {
}
+message VolumeEcShardReadRequest {
+ uint32 volume_id = 1;
+ uint32 shard_id = 2;
+ int64 offset = 3;
+ int64 size = 4;
+}
+message VolumeEcShardReadResponse {
+ bytes data = 1;
+}
+
message ReadVolumeFileStatusRequest {
uint32 volume_id = 1;
}
diff --git a/weed/pb/volume_server_pb/volume_server.pb.go b/weed/pb/volume_server_pb/volume_server.pb.go
index f572fe401..180fbc0af 100644
--- a/weed/pb/volume_server_pb/volume_server.pb.go
+++ b/weed/pb/volume_server_pb/volume_server.pb.go
@@ -53,6 +53,8 @@ It has these top-level messages:
VolumeEcShardsMountResponse
VolumeEcShardsUnmountRequest
VolumeEcShardsUnmountResponse
+ VolumeEcShardReadRequest
+ VolumeEcShardReadResponse
ReadVolumeFileStatusRequest
ReadVolumeFileStatusResponse
DiskStatus
@@ -811,7 +813,7 @@ func (*VolumeEcShardsGenerateResponse) Descriptor() ([]byte, []int) { return fil
type VolumeEcShardsCopyRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
Collection string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"`
- EcIndexes []uint32 `protobuf:"varint,3,rep,packed,name=ec_indexes,json=ecIndexes" json:"ec_indexes,omitempty"`
+ ShardIds []uint32 `protobuf:"varint,3,rep,packed,name=shard_ids,json=shardIds" json:"shard_ids,omitempty"`
SourceDataNode string `protobuf:"bytes,5,opt,name=source_data_node,json=sourceDataNode" json:"source_data_node,omitempty"`
}
@@ -834,9 +836,9 @@ func (m *VolumeEcShardsCopyRequest) GetCollection() string {
return ""
}
-func (m *VolumeEcShardsCopyRequest) GetEcIndexes() []uint32 {
+func (m *VolumeEcShardsCopyRequest) GetShardIds() []uint32 {
if m != nil {
- return m.EcIndexes
+ return m.ShardIds
}
return nil
}
@@ -859,7 +861,7 @@ func (*VolumeEcShardsCopyResponse) Descriptor() ([]byte, []int) { return fileDes
type VolumeEcShardsDeleteRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
ShouldDeleteEcx bool `protobuf:"varint,2,opt,name=should_delete_ecx,json=shouldDeleteEcx" json:"should_delete_ecx,omitempty"`
- EcIndexes []uint32 `protobuf:"varint,3,rep,packed,name=ec_indexes,json=ecIndexes" json:"ec_indexes,omitempty"`
+ ShardIds []uint32 `protobuf:"varint,3,rep,packed,name=shard_ids,json=shardIds" json:"shard_ids,omitempty"`
}
func (m *VolumeEcShardsDeleteRequest) Reset() { *m = VolumeEcShardsDeleteRequest{} }
@@ -881,9 +883,9 @@ func (m *VolumeEcShardsDeleteRequest) GetShouldDeleteEcx() bool {
return false
}
-func (m *VolumeEcShardsDeleteRequest) GetEcIndexes() []uint32 {
+func (m *VolumeEcShardsDeleteRequest) GetShardIds() []uint32 {
if m != nil {
- return m.EcIndexes
+ return m.ShardIds
}
return nil
}
@@ -899,7 +901,7 @@ func (*VolumeEcShardsDeleteResponse) Descriptor() ([]byte, []int) { return fileD
type VolumeEcShardsMountRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
Collection string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"`
- EcIndexes []uint32 `protobuf:"varint,3,rep,packed,name=ec_indexes,json=ecIndexes" json:"ec_indexes,omitempty"`
+ ShardIds []uint32 `protobuf:"varint,3,rep,packed,name=shard_ids,json=shardIds" json:"shard_ids,omitempty"`
}
func (m *VolumeEcShardsMountRequest) Reset() { *m = VolumeEcShardsMountRequest{} }
@@ -921,9 +923,9 @@ func (m *VolumeEcShardsMountRequest) GetCollection() string {
return ""
}
-func (m *VolumeEcShardsMountRequest) GetEcIndexes() []uint32 {
+func (m *VolumeEcShardsMountRequest) GetShardIds() []uint32 {
if m != nil {
- return m.EcIndexes
+ return m.ShardIds
}
return nil
}
@@ -937,8 +939,8 @@ func (*VolumeEcShardsMountResponse) ProtoMessage() {}
func (*VolumeEcShardsMountResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{41} }
type VolumeEcShardsUnmountRequest struct {
- VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
- EcIndexes []uint32 `protobuf:"varint,3,rep,packed,name=ec_indexes,json=ecIndexes" json:"ec_indexes,omitempty"`
+ VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
+ ShardIds []uint32 `protobuf:"varint,3,rep,packed,name=shard_ids,json=shardIds" json:"shard_ids,omitempty"`
}
func (m *VolumeEcShardsUnmountRequest) Reset() { *m = VolumeEcShardsUnmountRequest{} }
@@ -953,9 +955,9 @@ func (m *VolumeEcShardsUnmountRequest) GetVolumeId() uint32 {
return 0
}
-func (m *VolumeEcShardsUnmountRequest) GetEcIndexes() []uint32 {
+func (m *VolumeEcShardsUnmountRequest) GetShardIds() []uint32 {
if m != nil {
- return m.EcIndexes
+ return m.ShardIds
}
return nil
}
@@ -968,6 +970,62 @@ func (m *VolumeEcShardsUnmountResponse) String() string { return prot
func (*VolumeEcShardsUnmountResponse) ProtoMessage() {}
func (*VolumeEcShardsUnmountResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{43} }
+type VolumeEcShardReadRequest struct {
+ VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
+ ShardId uint32 `protobuf:"varint,2,opt,name=shard_id,json=shardId" json:"shard_id,omitempty"`
+ Offset int64 `protobuf:"varint,3,opt,name=offset" json:"offset,omitempty"`
+ Size int64 `protobuf:"varint,4,opt,name=size" json:"size,omitempty"`
+}
+
+func (m *VolumeEcShardReadRequest) Reset() { *m = VolumeEcShardReadRequest{} }
+func (m *VolumeEcShardReadRequest) String() string { return proto.CompactTextString(m) }
+func (*VolumeEcShardReadRequest) ProtoMessage() {}
+func (*VolumeEcShardReadRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{44} }
+
+func (m *VolumeEcShardReadRequest) GetVolumeId() uint32 {
+ if m != nil {
+ return m.VolumeId
+ }
+ return 0
+}
+
+func (m *VolumeEcShardReadRequest) GetShardId() uint32 {
+ if m != nil {
+ return m.ShardId
+ }
+ return 0
+}
+
+func (m *VolumeEcShardReadRequest) GetOffset() int64 {
+ if m != nil {
+ return m.Offset
+ }
+ return 0
+}
+
+func (m *VolumeEcShardReadRequest) GetSize() int64 {
+ if m != nil {
+ return m.Size
+ }
+ return 0
+}
+
+type VolumeEcShardReadResponse struct {
+ Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"`
+}
+
+func (m *VolumeEcShardReadResponse) Reset() { *m = VolumeEcShardReadResponse{} }
+func (m *VolumeEcShardReadResponse) String() string { return proto.CompactTextString(m) }
+func (*VolumeEcShardReadResponse) ProtoMessage() {}
+func (*VolumeEcShardReadResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{45} }
+
+func (m *VolumeEcShardReadResponse) GetData() []byte {
+ if m != nil {
+ return m.Data
+ }
+ return nil
+}
+
type ReadVolumeFileStatusRequest struct {
VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
}
@@ -975,7 +1033,7 @@ type ReadVolumeFileStatusRequest struct {
func (m *ReadVolumeFileStatusRequest) Reset() { *m = ReadVolumeFileStatusRequest{} }
func (m *ReadVolumeFileStatusRequest) String() string { return proto.CompactTextString(m) }
func (*ReadVolumeFileStatusRequest) ProtoMessage() {}
-func (*ReadVolumeFileStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{44} }
+func (*ReadVolumeFileStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{46} }
func (m *ReadVolumeFileStatusRequest) GetVolumeId() uint32 {
if m != nil {
@@ -998,7 +1056,7 @@ type ReadVolumeFileStatusResponse struct {
func (m *ReadVolumeFileStatusResponse) Reset() { *m = ReadVolumeFileStatusResponse{} }
func (m *ReadVolumeFileStatusResponse) String() string { return proto.CompactTextString(m) }
func (*ReadVolumeFileStatusResponse) ProtoMessage() {}
-func (*ReadVolumeFileStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{45} }
+func (*ReadVolumeFileStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{47} }
func (m *ReadVolumeFileStatusResponse) GetVolumeId() uint32 {
if m != nil {
@@ -1066,7 +1124,7 @@ type DiskStatus struct {
func (m *DiskStatus) Reset() { *m = DiskStatus{} }
func (m *DiskStatus) String() string { return proto.CompactTextString(m) }
func (*DiskStatus) ProtoMessage() {}
-func (*DiskStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{46} }
+func (*DiskStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{48} }
func (m *DiskStatus) GetDir() string {
if m != nil {
@@ -1109,7 +1167,7 @@ type MemStatus struct {
func (m *MemStatus) Reset() { *m = MemStatus{} }
func (m *MemStatus) String() string { return proto.CompactTextString(m) }
func (*MemStatus) ProtoMessage() {}
-func (*MemStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{47} }
+func (*MemStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{49} }
func (m *MemStatus) GetGoroutines() int32 {
if m != nil {
@@ -1205,6 +1263,8 @@ func init() {
proto.RegisterType((*VolumeEcShardsMountResponse)(nil), "volume_server_pb.VolumeEcShardsMountResponse")
proto.RegisterType((*VolumeEcShardsUnmountRequest)(nil), "volume_server_pb.VolumeEcShardsUnmountRequest")
proto.RegisterType((*VolumeEcShardsUnmountResponse)(nil), "volume_server_pb.VolumeEcShardsUnmountResponse")
+ proto.RegisterType((*VolumeEcShardReadRequest)(nil), "volume_server_pb.VolumeEcShardReadRequest")
+ proto.RegisterType((*VolumeEcShardReadResponse)(nil), "volume_server_pb.VolumeEcShardReadResponse")
proto.RegisterType((*ReadVolumeFileStatusRequest)(nil), "volume_server_pb.ReadVolumeFileStatusRequest")
proto.RegisterType((*ReadVolumeFileStatusResponse)(nil), "volume_server_pb.ReadVolumeFileStatusResponse")
proto.RegisterType((*DiskStatus)(nil), "volume_server_pb.DiskStatus")
@@ -1247,6 +1307,7 @@ type VolumeServerClient interface {
VolumeEcShardsDelete(ctx context.Context, in *VolumeEcShardsDeleteRequest, opts ...grpc.CallOption) (*VolumeEcShardsDeleteResponse, error)
VolumeEcShardsMount(ctx context.Context, in *VolumeEcShardsMountRequest, opts ...grpc.CallOption) (*VolumeEcShardsMountResponse, error)
VolumeEcShardsUnmount(ctx context.Context, in *VolumeEcShardsUnmountRequest, opts ...grpc.CallOption) (*VolumeEcShardsUnmountResponse, error)
+ VolumeEcShardRead(ctx context.Context, in *VolumeEcShardReadRequest, opts ...grpc.CallOption) (VolumeServer_VolumeEcShardReadClient, error)
}
type volumeServerClient struct {
@@ -1524,6 +1585,38 @@ func (c *volumeServerClient) VolumeEcShardsUnmount(ctx context.Context, in *Volu
return out, nil
}
+func (c *volumeServerClient) VolumeEcShardRead(ctx context.Context, in *VolumeEcShardReadRequest, opts ...grpc.CallOption) (VolumeServer_VolumeEcShardReadClient, error) {
+ stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[3], c.cc, "/volume_server_pb.VolumeServer/VolumeEcShardRead", opts...)
+ if err != nil {
+ return nil, err
+ }
+ x := &volumeServerVolumeEcShardReadClient{stream}
+ if err := x.ClientStream.SendMsg(in); err != nil {
+ return nil, err
+ }
+ if err := x.ClientStream.CloseSend(); err != nil {
+ return nil, err
+ }
+ return x, nil
+}
+
+type VolumeServer_VolumeEcShardReadClient interface {
+ Recv() (*VolumeEcShardReadResponse, error)
+ grpc.ClientStream
+}
+
+type volumeServerVolumeEcShardReadClient struct {
+ grpc.ClientStream
+}
+
+func (x *volumeServerVolumeEcShardReadClient) Recv() (*VolumeEcShardReadResponse, error) {
+ m := new(VolumeEcShardReadResponse)
+ if err := x.ClientStream.RecvMsg(m); err != nil {
+ return nil, err
+ }
+ return m, nil
+}
+
// Server API for VolumeServer service
type VolumeServerServer interface {
@@ -1552,6 +1645,7 @@ type VolumeServerServer interface {
VolumeEcShardsDelete(context.Context, *VolumeEcShardsDeleteRequest) (*VolumeEcShardsDeleteResponse, error)
VolumeEcShardsMount(context.Context, *VolumeEcShardsMountRequest) (*VolumeEcShardsMountResponse, error)
VolumeEcShardsUnmount(context.Context, *VolumeEcShardsUnmountRequest) (*VolumeEcShardsUnmountResponse, error)
+ VolumeEcShardRead(*VolumeEcShardReadRequest, VolumeServer_VolumeEcShardReadServer) error
}
func RegisterVolumeServerServer(s *grpc.Server, srv VolumeServerServer) {
@@ -1963,6 +2057,27 @@ func _VolumeServer_VolumeEcShardsUnmount_Handler(srv interface{}, ctx context.Co
return interceptor(ctx, in, info, handler)
}
+func _VolumeServer_VolumeEcShardRead_Handler(srv interface{}, stream grpc.ServerStream) error {
+ m := new(VolumeEcShardReadRequest)
+ if err := stream.RecvMsg(m); err != nil {
+ return err
+ }
+ return srv.(VolumeServerServer).VolumeEcShardRead(m, &volumeServerVolumeEcShardReadServer{stream})
+}
+
+type VolumeServer_VolumeEcShardReadServer interface {
+ Send(*VolumeEcShardReadResponse) error
+ grpc.ServerStream
+}
+
+type volumeServerVolumeEcShardReadServer struct {
+ grpc.ServerStream
+}
+
+func (x *volumeServerVolumeEcShardReadServer) Send(m *VolumeEcShardReadResponse) error {
+ return x.ServerStream.SendMsg(m)
+}
+
var _VolumeServer_serviceDesc = grpc.ServiceDesc{
ServiceName: "volume_server_pb.VolumeServer",
HandlerType: (*VolumeServerServer)(nil),
@@ -2060,6 +2175,11 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{
Handler: _VolumeServer_VolumeTailSender_Handler,
ServerStreams: true,
},
+ {
+ StreamName: "VolumeEcShardRead",
+ Handler: _VolumeServer_VolumeEcShardRead_Handler,
+ ServerStreams: true,
+ },
},
Metadata: "volume_server.proto",
}
@@ -2067,109 +2187,113 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{
func init() { proto.RegisterFile("volume_server.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
- // 1655 bytes of a gzipped FileDescriptorProto
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xc4, 0x59, 0x5b, 0x4f, 0xdc, 0xd6,
- 0x16, 0xc6, 0x99, 0x01, 0x86, 0x35, 0x90, 0xc0, 0x86, 0xc0, 0x60, 0x02, 0x21, 0x4e, 0x4e, 0x32,
- 0xb9, 0x41, 0x4e, 0xa2, 0x73, 0x4e, 0xce, 0x39, 0x0f, 0x6d, 0x42, 0xd2, 0x16, 0xa9, 0x49, 0x24,
- 0x93, 0x44, 0xbd, 0x44, 0xb2, 0x36, 0xf6, 0x22, 0x58, 0x78, 0xec, 0x89, 0xbd, 0x4d, 0x21, 0x6a,
- 0x9f, 0xda, 0xaa, 0xef, 0xfd, 0x07, 0x55, 0xdf, 0xfa, 0xd0, 0xd7, 0xfe, 0xaa, 0xfe, 0x82, 0xbe,
- 0x54, 0xfb, 0x62, 0xcf, 0xf8, 0xc6, 0x98, 0x26, 0x52, 0xdf, 0xf6, 0xac, 0xbd, 0xee, 0x5e, 0x97,
- 0xfd, 0x01, 0xcc, 0x1f, 0x06, 0x5e, 0xdc, 0x43, 0x2b, 0xc2, 0xf0, 0x10, 0xc3, 0x8d, 0x7e, 0x18,
- 0xb0, 0x80, 0xcc, 0x66, 0x88, 0x56, 0x7f, 0xd7, 0xd8, 0x04, 0xf2, 0x90, 0x32, 0x7b, 0xff, 0x11,
- 0x7a, 0xc8, 0xd0, 0xc4, 0x37, 0x31, 0x46, 0x8c, 0x2c, 0x43, 0x6b, 0xcf, 0xf5, 0xd0, 0x72, 0x9d,
- 0xa8, 0xa3, 0xad, 0x37, 0xba, 0x53, 0xe6, 0x24, 0xff, 0xbd, 0xed, 0x44, 0xc6, 0x33, 0x98, 0xcf,
- 0x08, 0x44, 0xfd, 0xc0, 0x8f, 0x90, 0xdc, 0x87, 0xc9, 0x10, 0xa3, 0xd8, 0x63, 0x52, 0xa0, 0x7d,
- 0x77, 0x6d, 0x23, 0x6f, 0x6b, 0x23, 0x15, 0x89, 0x3d, 0x66, 0x26, 0xec, 0x86, 0x0b, 0xd3, 0xc3,
- 0x17, 0x64, 0x09, 0x26, 0x95, 0xed, 0x8e, 0xb6, 0xae, 0x75, 0xa7, 0xcc, 0x09, 0x69, 0x9a, 0x2c,
- 0xc2, 0x44, 0xc4, 0x28, 0x8b, 0xa3, 0xce, 0x99, 0x75, 0xad, 0x3b, 0x6e, 0xaa, 0x5f, 0x64, 0x01,
- 0xc6, 0x31, 0x0c, 0x83, 0xb0, 0xd3, 0x10, 0xec, 0xf2, 0x07, 0x21, 0xd0, 0x8c, 0xdc, 0xb7, 0xd8,
- 0x69, 0xae, 0x6b, 0xdd, 0x19, 0x53, 0x9c, 0x8d, 0x49, 0x18, 0x7f, 0xdc, 0xeb, 0xb3, 0x63, 0xe3,
- 0x3f, 0xd0, 0x79, 0x49, 0xed, 0x38, 0xee, 0xbd, 0x14, 0x3e, 0x6e, 0xed, 0xa3, 0x7d, 0x90, 0xc4,
- 0xbe, 0x02, 0x53, 0xca, 0x73, 0xe5, 0xc1, 0x8c, 0xd9, 0x92, 0x84, 0x6d, 0xc7, 0xf8, 0x10, 0x96,
- 0x4b, 0x04, 0x55, 0x0e, 0x2e, 0xc3, 0xcc, 0x6b, 0x1a, 0xee, 0xd2, 0xd7, 0x68, 0x85, 0x94, 0xb9,
- 0x81, 0x90, 0xd6, 0xcc, 0x69, 0x45, 0x34, 0x39, 0xcd, 0xf8, 0x12, 0xf4, 0x8c, 0x86, 0xa0, 0xd7,
- 0xa7, 0x36, 0xab, 0x63, 0x9c, 0xac, 0x43, 0xbb, 0x1f, 0x22, 0xf5, 0xbc, 0xc0, 0xa6, 0x0c, 0x45,
- 0x16, 0x1a, 0xe6, 0x30, 0xc9, 0x58, 0x85, 0x95, 0x52, 0xe5, 0xd2, 0x41, 0xe3, 0x7e, 0xce, 0xfb,
- 0xa0, 0xd7, 0x73, 0x6b, 0x99, 0x36, 0x2e, 0x14, 0xbc, 0x16, 0x92, 0x4a, 0xef, 0x7f, 0x73, 0xb7,
- 0x1e, 0x52, 0x3f, 0xee, 0xd7, 0x52, 0x9c, 0xf7, 0x38, 0x11, 0x4d, 0x35, 0x2f, 0xc9, 0xe2, 0xd8,
- 0x0a, 0x3c, 0x0f, 0x6d, 0xe6, 0x06, 0x7e, 0xa2, 0x76, 0x0d, 0xc0, 0x4e, 0x89, 0xaa, 0x54, 0x86,
- 0x28, 0x86, 0x0e, 0x9d, 0xa2, 0xa8, 0x52, 0xfb, 0x8b, 0x06, 0xe7, 0x1f, 0xa8, 0xa4, 0x49, 0xc3,
- 0xb5, 0x3e, 0x40, 0xd6, 0xe4, 0x99, 0xbc, 0xc9, 0xfc, 0x07, 0x6a, 0x14, 0x3e, 0x10, 0xe7, 0x08,
- 0xb1, 0xef, 0xb9, 0x36, 0x15, 0x2a, 0x9a, 0x42, 0xc5, 0x30, 0x89, 0xcc, 0x42, 0x83, 0x31, 0xaf,
- 0x33, 0x2e, 0x6e, 0xf8, 0xd1, 0xe8, 0xc0, 0x62, 0xde, 0x57, 0x15, 0xc6, 0xbf, 0x61, 0x49, 0x52,
- 0x76, 0x8e, 0x7d, 0x7b, 0x47, 0x74, 0x43, 0xad, 0xa4, 0xff, 0xa1, 0x41, 0xa7, 0x28, 0xa8, 0xaa,
- 0xf8, 0x5d, 0x33, 0x70, 0xda, 0xf8, 0xc8, 0x45, 0x68, 0x33, 0xea, 0x7a, 0x56, 0xb0, 0xb7, 0x17,
- 0x21, 0xeb, 0x4c, 0xac, 0x6b, 0xdd, 0xa6, 0x09, 0x9c, 0xf4, 0x4c, 0x50, 0xc8, 0x75, 0x98, 0xb5,
- 0x65, 0x25, 0x5b, 0x21, 0x1e, 0xba, 0x11, 0xd7, 0x3c, 0x29, 0x1c, 0x3b, 0x67, 0x27, 0x15, 0x2e,
- 0xc9, 0xc4, 0x80, 0x19, 0xd7, 0x39, 0xb2, 0xc4, 0x00, 0x11, 0xed, 0xdf, 0x12, 0xda, 0xda, 0xae,
- 0x73, 0xf4, 0x91, 0xeb, 0xe1, 0x0e, 0x9f, 0x02, 0x2f, 0xe1, 0x82, 0x0c, 0x7e, 0xdb, 0xb7, 0x43,
- 0xec, 0xa1, 0xcf, 0xa8, 0xb7, 0x15, 0xf4, 0x8f, 0x6b, 0x95, 0xc0, 0x32, 0xb4, 0x22, 0xd7, 0xb7,
- 0xd1, 0xf2, 0xe5, 0x18, 0x6a, 0x9a, 0x93, 0xe2, 0xf7, 0xd3, 0xc8, 0x78, 0x08, 0xab, 0x15, 0x7a,
- 0x55, 0x66, 0x2f, 0xc1, 0xb4, 0x70, 0xcc, 0x0e, 0x7c, 0x86, 0x3e, 0x13, 0xba, 0xa7, 0xcd, 0x36,
- 0xa7, 0x6d, 0x49, 0x92, 0xf1, 0x4f, 0x20, 0x52, 0xc7, 0x93, 0x20, 0xf6, 0xeb, 0xb5, 0xe6, 0x79,
- 0x98, 0xcf, 0x88, 0xa8, 0xda, 0xb8, 0x07, 0x0b, 0x92, 0xfc, 0xc2, 0xef, 0xd5, 0xd6, 0xb5, 0x04,
- 0xe7, 0x73, 0x42, 0x4a, 0xdb, 0xdd, 0xc4, 0x48, 0x76, 0x4f, 0x9c, 0xa8, 0x6c, 0x31, 0xf1, 0x20,
- 0xbb, 0x2a, 0x8c, 0x5f, 0x35, 0x98, 0x4b, 0xc6, 0x48, 0xcd, 0xac, 0x9f, 0xb2, 0xec, 0x1a, 0x95,
- 0x65, 0xd7, 0x1c, 0x94, 0x5d, 0x17, 0x66, 0xa3, 0x20, 0x0e, 0x6d, 0xb4, 0x1c, 0xca, 0xa8, 0xe5,
- 0x07, 0x0e, 0xaa, 0xaa, 0x3c, 0x2b, 0xe9, 0x8f, 0x28, 0xa3, 0x4f, 0x03, 0x07, 0x8d, 0x0f, 0x92,
- 0x8f, 0x92, 0xf9, 0x9a, 0xd7, 0x61, 0xce, 0xa3, 0x11, 0xb3, 0x68, 0xbf, 0x8f, 0xbe, 0x63, 0x51,
- 0xc6, 0x4b, 0x42, 0x13, 0x25, 0x71, 0x96, 0x5f, 0x3c, 0x10, 0xf4, 0x07, 0xec, 0x69, 0x64, 0xfc,
- 0xa8, 0xc1, 0x39, 0x2e, 0xcb, 0x4b, 0xb0, 0x56, 0xbc, 0xb3, 0xd0, 0xc0, 0x23, 0xa6, 0x02, 0xe5,
- 0x47, 0xb2, 0x09, 0xf3, 0xaa, 0xd6, 0xdd, 0xc0, 0x1f, 0xb4, 0x41, 0x43, 0x08, 0x92, 0xc1, 0x55,
- 0xda, 0x09, 0x17, 0xa1, 0x1d, 0xb1, 0xa0, 0x9f, 0x74, 0x55, 0x53, 0x76, 0x15, 0x27, 0xc9, 0xae,
- 0x32, 0xfe, 0x05, 0xb3, 0x03, 0x9f, 0xea, 0x57, 0xe8, 0xb7, 0x5a, 0x32, 0x74, 0x9e, 0x53, 0xd7,
- 0xdb, 0x41, 0xdf, 0xc1, 0xf0, 0x1d, 0x3b, 0x87, 0xdc, 0x81, 0x05, 0xd7, 0xf1, 0xd0, 0x62, 0x6e,
- 0x0f, 0x83, 0x98, 0x59, 0x11, 0xda, 0x81, 0xef, 0x44, 0x49, 0x74, 0xfc, 0xee, 0xb9, 0xbc, 0xda,
- 0x91, 0x37, 0xc6, 0x77, 0xe9, 0x04, 0x1b, 0xf6, 0x62, 0xb0, 0x87, 0x7d, 0x44, 0xae, 0x70, 0x1f,
- 0xa9, 0x83, 0xa1, 0x0a, 0x63, 0x5a, 0x12, 0x3f, 0x11, 0x34, 0x9e, 0x1f, 0xc5, 0xb4, 0x1b, 0x38,
- 0xc7, 0xc2, 0xa3, 0x69, 0x13, 0x24, 0xe9, 0x61, 0xe0, 0x1c, 0x8b, 0x51, 0x12, 0x59, 0xe2, 0x13,
- 0xdb, 0xfb, 0xb1, 0x7f, 0x20, 0xbc, 0x69, 0x99, 0x6d, 0x37, 0xfa, 0x94, 0x46, 0x6c, 0x8b, 0x93,
- 0x8c, 0xdf, 0x34, 0x58, 0x1e, 0xb8, 0x61, 0xa2, 0x8d, 0xee, 0xe1, 0xdf, 0x90, 0x0e, 0x2e, 0xa1,
- 0x6a, 0x39, 0xf3, 0xe8, 0x52, 0xe5, 0x4e, 0xe4, 0x9d, 0x9a, 0xf8, 0xe2, 0x46, 0x2c, 0xf4, 0x12,
- 0xc7, 0x55, 0x8b, 0xbe, 0x4a, 0x46, 0xd9, 0x63, 0x7b, 0x67, 0x9f, 0x86, 0x4e, 0xf4, 0x31, 0xfa,
- 0x18, 0x52, 0xf6, 0x5e, 0xd6, 0xa4, 0xb1, 0x0e, 0x6b, 0x55, 0xda, 0x95, 0xfd, 0x9f, 0xd2, 0xbc,
- 0x26, 0x2c, 0xef, 0x6d, 0x54, 0xac, 0x02, 0xa0, 0x6d, 0xb9, 0xbe, 0x83, 0x47, 0xc8, 0x53, 0xda,
- 0xe8, 0xce, 0x98, 0x53, 0x68, 0x6f, 0x4b, 0xc2, 0x29, 0xa6, 0x42, 0x9a, 0xc1, 0xac, 0x8b, 0x2a,
- 0x82, 0xef, 0x35, 0x58, 0xc9, 0x5e, 0xd7, 0x9f, 0x9c, 0xe4, 0x06, 0xcc, 0x45, 0xfb, 0x41, 0xec,
- 0x39, 0x96, 0x23, 0x84, 0x2c, 0xb4, 0x8f, 0x44, 0x28, 0x2d, 0xf3, 0x9c, 0xbc, 0x90, 0xca, 0x1e,
- 0xdb, 0x47, 0x23, 0xe2, 0x31, 0xd6, 0x92, 0x65, 0x97, 0x77, 0x43, 0xf9, 0x79, 0x94, 0x8f, 0xa2,
- 0xf6, 0xe2, 0x79, 0xc7, 0x4c, 0x8b, 0x97, 0x5f, 0x99, 0x65, 0xe5, 0xd8, 0x17, 0x79, 0xc7, 0x4f,
- 0xb1, 0xc7, 0x46, 0x99, 0xbe, 0x98, 0x2f, 0xef, 0xfc, 0xba, 0xfb, 0x1f, 0xac, 0x98, 0x48, 0x1d,
- 0xc9, 0x24, 0x1e, 0x0e, 0xf5, 0x1f, 0x57, 0xbf, 0x9f, 0x81, 0x0b, 0xe5, 0xc2, 0x75, 0x1e, 0x58,
- 0xff, 0x07, 0x3d, 0x7d, 0xc0, 0xf0, 0xfe, 0x8f, 0x18, 0xed, 0xf5, 0xd3, 0x09, 0x20, 0x07, 0xc5,
- 0x92, 0x7a, 0xcd, 0x3c, 0x4f, 0xee, 0x93, 0x31, 0x50, 0x78, 0xfd, 0x34, 0x0a, 0xaf, 0x1f, 0x6e,
- 0xc0, 0xa1, 0xac, 0xca, 0x80, 0x5c, 0x13, 0x4b, 0x0e, 0x65, 0x55, 0x06, 0x52, 0x61, 0x61, 0x60,
- 0x5c, 0x1a, 0x50, 0xfc, 0xc2, 0xc0, 0x2a, 0x80, 0xda, 0x21, 0xb1, 0x9f, 0xbc, 0xe6, 0xa6, 0xe4,
- 0x06, 0x89, 0xfd, 0xca, 0x45, 0x36, 0x59, 0xb9, 0xc8, 0xb2, 0x65, 0xd6, 0x2a, 0x4c, 0x93, 0xcf,
- 0x00, 0x1e, 0xb9, 0xd1, 0x81, 0x4c, 0x32, 0xdf, 0x9c, 0x8e, 0x1b, 0x2a, 0x38, 0xc0, 0x8f, 0x9c,
- 0x42, 0x3d, 0x4f, 0xa5, 0x8e, 0x1f, 0x39, 0x34, 0x8c, 0x23, 0x74, 0x54, 0x76, 0xc4, 0x99, 0xd3,
- 0xf6, 0x42, 0x44, 0x95, 0x00, 0x71, 0x36, 0x7e, 0xd6, 0x60, 0xea, 0x09, 0xf6, 0x94, 0xe6, 0x35,
- 0x80, 0xd7, 0x41, 0x18, 0xc4, 0xcc, 0xf5, 0x51, 0x2e, 0xfa, 0x71, 0x73, 0x88, 0xf2, 0xd7, 0xed,
- 0x08, 0xa8, 0x8a, 0xde, 0x9e, 0x4a, 0xa6, 0x38, 0x73, 0xda, 0x3e, 0xd2, 0xbe, 0xca, 0x9f, 0x38,
- 0x73, 0xa0, 0x1b, 0x31, 0x6a, 0x1f, 0x88, 0x64, 0x35, 0x4d, 0xf9, 0xe3, 0xee, 0x0f, 0xf3, 0x30,
- 0x3d, 0x3c, 0xda, 0xc9, 0x2b, 0x68, 0x0f, 0x21, 0x74, 0x72, 0xa5, 0x08, 0xc4, 0x8b, 0x88, 0x5f,
- 0xff, 0xc7, 0x08, 0x2e, 0xd5, 0x18, 0x63, 0xc4, 0x87, 0xb9, 0x02, 0x02, 0x26, 0x37, 0x8a, 0xd2,
- 0x55, 0xf8, 0x5a, 0xbf, 0x59, 0x8b, 0x37, 0xb5, 0xc7, 0x60, 0xbe, 0x04, 0xd2, 0x92, 0x5b, 0x23,
- 0xb4, 0x64, 0x60, 0xb5, 0x7e, 0xbb, 0x26, 0x77, 0x6a, 0xf5, 0x0d, 0x90, 0x22, 0xde, 0x25, 0x37,
- 0x47, 0xaa, 0x19, 0xe0, 0x69, 0xfd, 0x56, 0x3d, 0xe6, 0xca, 0x40, 0x25, 0x12, 0x1e, 0x19, 0x68,
- 0x06, 0x6b, 0x8f, 0x0c, 0x34, 0x07, 0xaf, 0xc7, 0xc8, 0x01, 0xcc, 0xe6, 0x51, 0x32, 0xb9, 0x5e,
- 0xf5, 0xa7, 0x9b, 0x02, 0x08, 0xd7, 0x6f, 0xd4, 0x61, 0x4d, 0x8d, 0x21, 0x9c, 0xcd, 0x22, 0x59,
- 0x72, 0xad, 0x28, 0x5f, 0x8a, 0xcb, 0xf5, 0xee, 0x68, 0xc6, 0xe1, 0x98, 0xf2, 0xe8, 0xb6, 0x2c,
- 0xa6, 0x0a, 0xe8, 0x5c, 0x16, 0x53, 0x15, 0x58, 0x36, 0xc6, 0xc8, 0xd7, 0x09, 0x64, 0xca, 0xa1,
- 0x3e, 0xb2, 0x51, 0xa5, 0xa6, 0x1c, 0x76, 0xea, 0x9b, 0xb5, 0xf9, 0x13, 0xdb, 0x77, 0x34, 0xde,
- 0xeb, 0x43, 0xe0, 0xaf, 0xac, 0xd7, 0x8b, 0x70, 0xb2, 0xac, 0xd7, 0xcb, 0x10, 0xe4, 0x18, 0xd9,
- 0x85, 0x99, 0x0c, 0x1c, 0x24, 0x57, 0xab, 0x24, 0xb3, 0xcb, 0x59, 0xbf, 0x36, 0x92, 0x2f, 0xb5,
- 0x61, 0x25, 0xd3, 0x4b, 0x8d, 0xab, 0x4a, 0xe7, 0xb2, 0xf3, 0xea, 0xea, 0x28, 0xb6, 0xd4, 0xc0,
- 0xe7, 0x00, 0x03, 0xf4, 0x46, 0x2e, 0x57, 0xc9, 0x0d, 0x7f, 0x8a, 0x2b, 0x27, 0x33, 0xa5, 0xaa,
- 0xbf, 0x82, 0x85, 0xb2, 0x4d, 0x4f, 0x4a, 0xba, 0xf0, 0x84, 0xe7, 0x84, 0xbe, 0x51, 0x97, 0x3d,
- 0x35, 0xfc, 0x02, 0x5a, 0x09, 0x76, 0x23, 0x97, 0x8a, 0xd2, 0x39, 0xac, 0xa9, 0x1b, 0x27, 0xb1,
- 0x0c, 0x55, 0x53, 0x2f, 0x69, 0x9c, 0x01, 0xa8, 0xaa, 0x6e, 0x9c, 0x02, 0xfc, 0xab, 0x6e, 0x9c,
- 0x22, 0x46, 0x13, 0xe6, 0xde, 0x24, 0xb8, 0x7a, 0x18, 0x83, 0x94, 0x0e, 0xd9, 0x2a, 0x88, 0x55,
- 0x3a, 0x64, 0xab, 0x61, 0xcd, 0x18, 0xf9, 0x06, 0x16, 0xcb, 0xa1, 0x07, 0xa9, 0x6c, 0xbf, 0x0a,
- 0x08, 0xa4, 0xdf, 0xa9, 0x2f, 0x90, 0x59, 0x2b, 0x05, 0xcc, 0x50, 0x1d, 0x71, 0x09, 0xf8, 0xa9,
- 0x8e, 0xb8, 0x14, 0x86, 0x88, 0x1a, 0x2d, 0x03, 0x00, 0x65, 0x35, 0x7a, 0x02, 0x5e, 0xd1, 0x37,
- 0xea, 0xb2, 0x67, 0xf6, 0x59, 0xf1, 0x7d, 0x4f, 0x46, 0xfa, 0x9f, 0x19, 0x55, 0xb7, 0x6b, 0x72,
- 0xa7, 0x56, 0xdf, 0x26, 0xe3, 0x38, 0xf7, 0xb4, 0x27, 0x23, 0x03, 0xc8, 0x8d, 0xb0, 0xcd, 0xda,
- 0xfc, 0x89, 0xed, 0xdd, 0x09, 0xf1, 0x4f, 0x96, 0x7b, 0x7f, 0x06, 0x00, 0x00, 0xff, 0xff, 0xb2,
- 0x60, 0x85, 0x8e, 0x7b, 0x19, 0x00, 0x00,
+ // 1724 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xc4, 0x59, 0xdd, 0x6e, 0xd4, 0xc6,
+ 0x17, 0x8f, 0xd9, 0x4d, 0x76, 0x73, 0x36, 0x81, 0x64, 0x12, 0x92, 0x8d, 0x03, 0x61, 0x31, 0xfc,
+ 0x21, 0x04, 0x48, 0xf8, 0x83, 0xda, 0xd2, 0xf6, 0xa2, 0x85, 0x40, 0xdb, 0x48, 0x05, 0x24, 0x07,
+ 0x10, 0x55, 0x91, 0xac, 0x89, 0x3d, 0x21, 0x56, 0xbc, 0xf6, 0xe2, 0x19, 0xa7, 0x09, 0x6a, 0x7b,
+ 0x43, 0x9f, 0xa0, 0x2f, 0xd0, 0x8b, 0xde, 0xf5, 0xa2, 0xb7, 0x7d, 0xaa, 0x3e, 0x40, 0xd5, 0x9b,
+ 0x6a, 0x3e, 0xec, 0xf5, 0x67, 0xd6, 0x29, 0x48, 0xbd, 0x9b, 0x3d, 0x73, 0xbe, 0x7d, 0xce, 0x99,
+ 0xf9, 0xcd, 0xc2, 0xdc, 0x41, 0xe0, 0x45, 0x7d, 0x62, 0x51, 0x12, 0x1e, 0x90, 0x70, 0x7d, 0x10,
+ 0x06, 0x2c, 0x40, 0x33, 0x19, 0xa2, 0x35, 0xd8, 0x31, 0x36, 0x00, 0xdd, 0xc7, 0xcc, 0xde, 0x7b,
+ 0x40, 0x3c, 0xc2, 0x88, 0x49, 0x5e, 0x47, 0x84, 0x32, 0xb4, 0x04, 0xed, 0x5d, 0xd7, 0x23, 0x96,
+ 0xeb, 0xd0, 0xae, 0xd6, 0x6b, 0xac, 0x4e, 0x9a, 0x2d, 0xfe, 0x7b, 0xcb, 0xa1, 0xc6, 0x13, 0x98,
+ 0xcb, 0x08, 0xd0, 0x41, 0xe0, 0x53, 0x82, 0xee, 0x42, 0x2b, 0x24, 0x34, 0xf2, 0x98, 0x14, 0xe8,
+ 0xdc, 0x5e, 0x59, 0xcf, 0xdb, 0x5a, 0x4f, 0x44, 0x22, 0x8f, 0x99, 0x31, 0xbb, 0xe1, 0xc2, 0x54,
+ 0x7a, 0x03, 0x2d, 0x42, 0x4b, 0xd9, 0xee, 0x6a, 0x3d, 0x6d, 0x75, 0xd2, 0x9c, 0x90, 0xa6, 0xd1,
+ 0x02, 0x4c, 0x50, 0x86, 0x59, 0x44, 0xbb, 0xa7, 0x7a, 0xda, 0xea, 0xb8, 0xa9, 0x7e, 0xa1, 0x79,
+ 0x18, 0x27, 0x61, 0x18, 0x84, 0xdd, 0x86, 0x60, 0x97, 0x3f, 0x10, 0x82, 0x26, 0x75, 0xdf, 0x90,
+ 0x6e, 0xb3, 0xa7, 0xad, 0x4e, 0x9b, 0x62, 0x6d, 0xb4, 0x60, 0xfc, 0x61, 0x7f, 0xc0, 0x8e, 0x8c,
+ 0x8f, 0xa0, 0xfb, 0x1c, 0xdb, 0x51, 0xd4, 0x7f, 0x2e, 0x7c, 0xdc, 0xdc, 0x23, 0xf6, 0x7e, 0x1c,
+ 0xfb, 0x32, 0x4c, 0x2a, 0xcf, 0x95, 0x07, 0xd3, 0x66, 0x5b, 0x12, 0xb6, 0x1c, 0xe3, 0x73, 0x58,
+ 0x2a, 0x11, 0x54, 0x39, 0xb8, 0x04, 0xd3, 0xaf, 0x70, 0xb8, 0x83, 0x5f, 0x11, 0x2b, 0xc4, 0xcc,
+ 0x0d, 0x84, 0xb4, 0x66, 0x4e, 0x29, 0xa2, 0xc9, 0x69, 0xc6, 0xb7, 0xa0, 0x67, 0x34, 0x04, 0xfd,
+ 0x01, 0xb6, 0x59, 0x1d, 0xe3, 0xa8, 0x07, 0x9d, 0x41, 0x48, 0xb0, 0xe7, 0x05, 0x36, 0x66, 0x44,
+ 0x64, 0xa1, 0x61, 0xa6, 0x49, 0xc6, 0x79, 0x58, 0x2e, 0x55, 0x2e, 0x1d, 0x34, 0xee, 0xe6, 0xbc,
+ 0x0f, 0xfa, 0x7d, 0xb7, 0x96, 0x69, 0xe3, 0x5c, 0xc1, 0x6b, 0x21, 0xa9, 0xf4, 0x7e, 0x9c, 0xdb,
+ 0xf5, 0x08, 0xf6, 0xa3, 0x41, 0x2d, 0xc5, 0x79, 0x8f, 0x63, 0xd1, 0x44, 0xf3, 0xa2, 0x2c, 0x8e,
+ 0xcd, 0xc0, 0xf3, 0x88, 0xcd, 0xdc, 0xc0, 0x8f, 0xd5, 0xae, 0x00, 0xd8, 0x09, 0x51, 0x95, 0x4a,
+ 0x8a, 0x62, 0xe8, 0xd0, 0x2d, 0x8a, 0x2a, 0xb5, 0xbf, 0x69, 0x70, 0xf6, 0x9e, 0x4a, 0x9a, 0x34,
+ 0x5c, 0xeb, 0x03, 0x64, 0x4d, 0x9e, 0xca, 0x9b, 0xcc, 0x7f, 0xa0, 0x46, 0xe1, 0x03, 0x71, 0x8e,
+ 0x90, 0x0c, 0x3c, 0xd7, 0xc6, 0x42, 0x45, 0x53, 0xa8, 0x48, 0x93, 0xd0, 0x0c, 0x34, 0x18, 0xf3,
+ 0xba, 0xe3, 0x62, 0x87, 0x2f, 0x8d, 0x2e, 0x2c, 0xe4, 0x7d, 0x55, 0x61, 0x7c, 0x08, 0x8b, 0x92,
+ 0xb2, 0x7d, 0xe4, 0xdb, 0xdb, 0xa2, 0x1b, 0x6a, 0x25, 0xfd, 0x6f, 0x0d, 0xba, 0x45, 0x41, 0x55,
+ 0xc5, 0xef, 0x9a, 0x81, 0x93, 0xc6, 0x87, 0x2e, 0x40, 0x87, 0x61, 0xd7, 0xb3, 0x82, 0xdd, 0x5d,
+ 0x4a, 0x58, 0x77, 0xa2, 0xa7, 0xad, 0x36, 0x4d, 0xe0, 0xa4, 0x27, 0x82, 0x82, 0xae, 0xc1, 0x8c,
+ 0x2d, 0x2b, 0xd9, 0x0a, 0xc9, 0x81, 0x4b, 0xb9, 0xe6, 0x96, 0x70, 0xec, 0x8c, 0x1d, 0x57, 0xb8,
+ 0x24, 0x23, 0x03, 0xa6, 0x5d, 0xe7, 0xd0, 0x12, 0x03, 0x44, 0xb4, 0x7f, 0x5b, 0x68, 0xeb, 0xb8,
+ 0xce, 0xe1, 0x17, 0xae, 0x47, 0xb6, 0xf9, 0x14, 0x78, 0x0e, 0xe7, 0x64, 0xf0, 0x5b, 0xbe, 0x1d,
+ 0x92, 0x3e, 0xf1, 0x19, 0xf6, 0x36, 0x83, 0xc1, 0x51, 0xad, 0x12, 0x58, 0x82, 0x36, 0x75, 0x7d,
+ 0x9b, 0x58, 0xbe, 0x1c, 0x43, 0x4d, 0xb3, 0x25, 0x7e, 0x3f, 0xa6, 0xc6, 0x7d, 0x38, 0x5f, 0xa1,
+ 0x57, 0x65, 0xf6, 0x22, 0x4c, 0x09, 0xc7, 0xec, 0xc0, 0x67, 0xc4, 0x67, 0x42, 0xf7, 0x94, 0xd9,
+ 0xe1, 0xb4, 0x4d, 0x49, 0x32, 0xfe, 0x0f, 0x48, 0xea, 0x78, 0x14, 0x44, 0x7e, 0xbd, 0xd6, 0x3c,
+ 0x0b, 0x73, 0x19, 0x11, 0x55, 0x1b, 0x77, 0x60, 0x5e, 0x92, 0x9f, 0xf9, 0xfd, 0xda, 0xba, 0x16,
+ 0xe1, 0x6c, 0x4e, 0x48, 0x69, 0xbb, 0x1d, 0x1b, 0xc9, 0x9e, 0x13, 0xc7, 0x2a, 0x5b, 0x88, 0x3d,
+ 0xc8, 0x1e, 0x15, 0xc6, 0xef, 0x1a, 0xcc, 0xc6, 0x63, 0xa4, 0x66, 0xd6, 0x4f, 0x58, 0x76, 0x8d,
+ 0xca, 0xb2, 0x6b, 0x0e, 0xcb, 0x6e, 0x15, 0x66, 0x68, 0x10, 0x85, 0x36, 0xb1, 0x1c, 0xcc, 0xb0,
+ 0xe5, 0x07, 0x0e, 0x51, 0x55, 0x79, 0x5a, 0xd2, 0x1f, 0x60, 0x86, 0x1f, 0x07, 0x0e, 0x31, 0x3e,
+ 0x8b, 0x3f, 0x4a, 0xe6, 0x6b, 0x5e, 0x83, 0x59, 0x0f, 0x53, 0x66, 0xe1, 0xc1, 0x80, 0xf8, 0x8e,
+ 0x85, 0x19, 0x2f, 0x09, 0x4d, 0x94, 0xc4, 0x69, 0xbe, 0x71, 0x4f, 0xd0, 0xef, 0xb1, 0xc7, 0xd4,
+ 0xf8, 0x59, 0x83, 0x33, 0x5c, 0x96, 0x97, 0x60, 0xad, 0x78, 0x67, 0xa0, 0x41, 0x0e, 0x99, 0x0a,
+ 0x94, 0x2f, 0xd1, 0x06, 0xcc, 0xa9, 0x5a, 0x77, 0x03, 0x7f, 0xd8, 0x06, 0x0d, 0x21, 0x88, 0x86,
+ 0x5b, 0x49, 0x27, 0x5c, 0x80, 0x0e, 0x65, 0xc1, 0x20, 0xee, 0xaa, 0xa6, 0xec, 0x2a, 0x4e, 0x92,
+ 0x5d, 0x65, 0x7c, 0x00, 0x33, 0x43, 0x9f, 0xea, 0x57, 0xe8, 0x5b, 0x2d, 0x1e, 0x3a, 0x4f, 0xb1,
+ 0xeb, 0x6d, 0x13, 0xdf, 0x21, 0xe1, 0x3b, 0x76, 0x0e, 0xba, 0x05, 0xf3, 0xae, 0xe3, 0x11, 0x8b,
+ 0xb9, 0x7d, 0x12, 0x44, 0xcc, 0xa2, 0xc4, 0x0e, 0x7c, 0x87, 0xc6, 0xd1, 0xf1, 0xbd, 0xa7, 0x72,
+ 0x6b, 0x5b, 0xee, 0x18, 0x3f, 0x25, 0x13, 0x2c, 0xed, 0xc5, 0xf0, 0x1c, 0xf6, 0x09, 0xe1, 0x0a,
+ 0xf7, 0x08, 0x76, 0x48, 0xa8, 0xc2, 0x98, 0x92, 0xc4, 0xaf, 0x04, 0x8d, 0xe7, 0x47, 0x31, 0xed,
+ 0x04, 0xce, 0x91, 0xf0, 0x68, 0xca, 0x04, 0x49, 0xba, 0x1f, 0x38, 0x47, 0x62, 0x94, 0x50, 0x4b,
+ 0x7c, 0x62, 0x7b, 0x2f, 0xf2, 0xf7, 0x85, 0x37, 0x6d, 0xb3, 0xe3, 0xd2, 0xaf, 0x31, 0x65, 0x9b,
+ 0x9c, 0x64, 0xfc, 0xa1, 0xc1, 0xd2, 0xd0, 0x0d, 0x93, 0xd8, 0xc4, 0x3d, 0xf8, 0x0f, 0xd2, 0xc1,
+ 0x25, 0x54, 0x2d, 0x67, 0x2e, 0x5d, 0xaa, 0xdc, 0x91, 0xdc, 0x53, 0x13, 0x5f, 0xec, 0x88, 0x03,
+ 0xbd, 0xc4, 0x71, 0xd5, 0xa2, 0x2f, 0xe3, 0x51, 0xf6, 0xd0, 0xde, 0xde, 0xc3, 0xa1, 0x43, 0xbf,
+ 0x24, 0x3e, 0x09, 0x31, 0x7b, 0x2f, 0xc7, 0xa4, 0xd1, 0x83, 0x95, 0x2a, 0xed, 0xca, 0xfe, 0x2f,
+ 0x49, 0x5e, 0x63, 0x96, 0xf7, 0x36, 0x2a, 0x96, 0x61, 0x92, 0x72, 0x8d, 0xe2, 0x6e, 0xdb, 0xe8,
+ 0x35, 0xb8, 0xb0, 0x20, 0x6c, 0x39, 0xf4, 0x04, 0x33, 0x21, 0xc9, 0x5f, 0xd6, 0x41, 0xe5, 0xff,
+ 0x5b, 0x0d, 0x96, 0xb3, 0xdb, 0xf5, 0xe7, 0x26, 0x5a, 0x83, 0x59, 0xba, 0x17, 0x44, 0x9e, 0x63,
+ 0x39, 0x42, 0xc8, 0x22, 0xf6, 0xa1, 0x08, 0xa4, 0x6d, 0x9e, 0x91, 0x1b, 0x52, 0xd9, 0x43, 0xfb,
+ 0xf0, 0xd8, 0x68, 0x8c, 0x95, 0xf8, 0xa0, 0xcb, 0x3b, 0xa1, 0xbc, 0x3c, 0xc8, 0xc7, 0x50, 0xfb,
+ 0xd0, 0x79, 0xa7, 0x2c, 0x8b, 0x3b, 0x5f, 0x99, 0x5d, 0xe5, 0xd6, 0x8b, 0xbc, 0xdb, 0x27, 0x38,
+ 0xc1, 0x8e, 0x37, 0x7c, 0x21, 0x5f, 0xd6, 0xf9, 0x63, 0xee, 0xc7, 0x78, 0xaa, 0x28, 0x06, 0x93,
+ 0x60, 0xa7, 0x76, 0x37, 0x2b, 0xb3, 0x22, 0x1b, 0xd3, 0x66, 0x4b, 0x59, 0xe5, 0xb0, 0x45, 0xcd,
+ 0x60, 0x79, 0x1f, 0x54, 0xbf, 0x32, 0x00, 0xa5, 0xa1, 0x00, 0xca, 0x46, 0xae, 0xec, 0xa5, 0x7d,
+ 0x35, 0xd6, 0x10, 0x34, 0x79, 0x55, 0xaa, 0x69, 0x26, 0xd6, 0xc6, 0x27, 0xb0, 0xcc, 0x79, 0xa4,
+ 0x90, 0xb8, 0xe1, 0xd4, 0xbf, 0x05, 0xfe, 0x79, 0x0a, 0xce, 0x95, 0x0b, 0xd7, 0xb9, 0x09, 0x7e,
+ 0x0a, 0x7a, 0x72, 0xd3, 0xe2, 0x83, 0x8a, 0x32, 0xdc, 0x1f, 0x24, 0xa3, 0x4a, 0x4e, 0xb4, 0x45,
+ 0x75, 0xed, 0x7a, 0x1a, 0xef, 0xc7, 0xf3, 0xaa, 0x70, 0x4d, 0x6b, 0x14, 0xae, 0x69, 0xdc, 0x80,
+ 0x83, 0x59, 0x95, 0x01, 0x79, 0x9e, 0x2d, 0x3a, 0x98, 0x55, 0x19, 0x48, 0x84, 0x85, 0x81, 0x71,
+ 0x69, 0x40, 0xf1, 0x0b, 0x03, 0xe7, 0x01, 0xd4, 0x61, 0x17, 0xf9, 0xf1, 0xb5, 0x73, 0x52, 0x1e,
+ 0x75, 0x91, 0x5f, 0x79, 0xe2, 0xb6, 0x2a, 0x4f, 0xdc, 0x6c, 0x4f, 0xb4, 0x0b, 0x63, 0xef, 0x05,
+ 0xc0, 0x03, 0x97, 0xee, 0xcb, 0x24, 0xf3, 0x23, 0xde, 0x71, 0x43, 0x85, 0x5b, 0xf8, 0x92, 0x53,
+ 0xb0, 0xe7, 0xa9, 0xd4, 0xf1, 0x25, 0xff, 0xe2, 0x11, 0x25, 0x8e, 0xca, 0x8e, 0x58, 0x73, 0xda,
+ 0x6e, 0x48, 0x88, 0x4a, 0x80, 0x58, 0x1b, 0xbf, 0x6a, 0x30, 0xf9, 0x88, 0xf4, 0x95, 0xe6, 0x15,
+ 0x80, 0x57, 0x41, 0x18, 0x44, 0xcc, 0xf5, 0x89, 0xbc, 0x91, 0x8c, 0x9b, 0x29, 0xca, 0xbf, 0xb7,
+ 0x23, 0x4a, 0x96, 0x78, 0xbb, 0x2a, 0x99, 0x62, 0xcd, 0x69, 0x7b, 0x04, 0x0f, 0x54, 0xfe, 0xc4,
+ 0x9a, 0x23, 0x72, 0xca, 0xb0, 0xbd, 0x2f, 0x92, 0xd5, 0x34, 0xe5, 0x8f, 0xdb, 0x7f, 0xcd, 0xc1,
+ 0x54, 0xfa, 0x0c, 0x42, 0x2f, 0xa1, 0x93, 0x7a, 0x4a, 0x40, 0x97, 0x8b, 0x2f, 0x06, 0xc5, 0xa7,
+ 0x09, 0xfd, 0x7f, 0x23, 0xb8, 0x54, 0x27, 0x8f, 0x21, 0x1f, 0x66, 0x0b, 0x50, 0x1d, 0xad, 0x15,
+ 0xa5, 0xab, 0x1e, 0x02, 0xf4, 0xeb, 0xb5, 0x78, 0x13, 0x7b, 0x0c, 0xe6, 0x4a, 0xb0, 0x37, 0xba,
+ 0x31, 0x42, 0x4b, 0x06, 0xff, 0xeb, 0x37, 0x6b, 0x72, 0x27, 0x56, 0x5f, 0x03, 0x2a, 0x02, 0x73,
+ 0x74, 0x7d, 0xa4, 0x9a, 0x21, 0xf0, 0xd7, 0x6f, 0xd4, 0x63, 0xae, 0x0c, 0x54, 0x42, 0xf6, 0x91,
+ 0x81, 0x66, 0x1e, 0x05, 0x46, 0x06, 0x9a, 0x7b, 0x07, 0x18, 0x43, 0xfb, 0x30, 0x93, 0x87, 0xf3,
+ 0xe8, 0x5a, 0xd5, 0x1b, 0x53, 0xe1, 0xb5, 0x40, 0x5f, 0xab, 0xc3, 0x9a, 0x18, 0x23, 0x70, 0x3a,
+ 0x0b, 0xb9, 0xd1, 0xd5, 0xa2, 0x7c, 0xe9, 0x03, 0x82, 0xbe, 0x3a, 0x9a, 0x31, 0x1d, 0x53, 0x1e,
+ 0x86, 0x97, 0xc5, 0x54, 0x81, 0xf1, 0xcb, 0x62, 0xaa, 0x42, 0xf5, 0xc6, 0x18, 0xfa, 0x3e, 0xc6,
+ 0x76, 0x39, 0x78, 0x8a, 0xd6, 0xab, 0xd4, 0x94, 0xe3, 0x63, 0x7d, 0xa3, 0x36, 0x7f, 0x6c, 0xfb,
+ 0x96, 0xc6, 0x7b, 0x3d, 0x85, 0x52, 0xcb, 0x7a, 0xbd, 0x88, 0x7b, 0xcb, 0x7a, 0xbd, 0x0c, 0xea,
+ 0x8e, 0xa1, 0x1d, 0x98, 0xce, 0xe0, 0x56, 0x74, 0xa5, 0x4a, 0x32, 0x7b, 0x97, 0xd0, 0xaf, 0x8e,
+ 0xe4, 0x4b, 0x6c, 0x58, 0xf1, 0xf4, 0x52, 0xe3, 0xaa, 0xd2, 0xb9, 0xec, 0xbc, 0xba, 0x32, 0x8a,
+ 0x2d, 0x31, 0xf0, 0x0d, 0xc0, 0x10, 0x66, 0xa2, 0x4b, 0x55, 0x72, 0xe9, 0x4f, 0x71, 0xf9, 0x78,
+ 0xa6, 0x44, 0xf5, 0x77, 0x30, 0x5f, 0x76, 0xd2, 0xa3, 0x92, 0x2e, 0x3c, 0xe6, 0x3a, 0xa1, 0xaf,
+ 0xd7, 0x65, 0x4f, 0x0c, 0x3f, 0x83, 0x76, 0x0c, 0x32, 0xd1, 0xc5, 0xa2, 0x74, 0x0e, 0x14, 0xeb,
+ 0xc6, 0x71, 0x2c, 0xa9, 0x6a, 0xea, 0xc7, 0x8d, 0x33, 0x44, 0x7f, 0xd5, 0x8d, 0x53, 0xc0, 0xa9,
+ 0xd5, 0x8d, 0x53, 0x04, 0x93, 0xc2, 0xdc, 0xeb, 0xf8, 0x01, 0x20, 0x0d, 0x96, 0x4a, 0x87, 0x6c,
+ 0x15, 0x16, 0x2c, 0x1d, 0xb2, 0xd5, 0xf8, 0x6b, 0x0c, 0xfd, 0x00, 0x0b, 0xe5, 0x18, 0x09, 0x55,
+ 0xb6, 0x5f, 0x05, 0x56, 0xd3, 0x6f, 0xd5, 0x17, 0xc8, 0x1c, 0x2b, 0x05, 0x78, 0x53, 0x1d, 0x71,
+ 0x09, 0x4a, 0xab, 0x8e, 0xb8, 0x14, 0x31, 0x89, 0x1a, 0x2d, 0x43, 0x2b, 0x65, 0x35, 0x7a, 0x0c,
+ 0xb4, 0xd2, 0xd7, 0xeb, 0xb2, 0x67, 0xce, 0xb3, 0x22, 0x1c, 0x41, 0x23, 0xfd, 0xcf, 0x8c, 0xaa,
+ 0x9b, 0x35, 0xb9, 0x13, 0xab, 0x6f, 0xe2, 0x71, 0x9c, 0xc3, 0x22, 0x68, 0x64, 0x00, 0xb9, 0x11,
+ 0xb6, 0x51, 0x9b, 0x3f, 0xb1, 0x3d, 0x88, 0x1f, 0xe0, 0x52, 0x30, 0x03, 0xad, 0x8d, 0xd0, 0x93,
+ 0xc2, 0x42, 0xfa, 0xf5, 0x5a, 0xbc, 0xc3, 0x0e, 0xda, 0x99, 0x10, 0xff, 0x3f, 0xdd, 0xf9, 0x27,
+ 0x00, 0x00, 0xff, 0xff, 0x67, 0x03, 0x26, 0xea, 0x96, 0x1a, 0x00, 0x00,
}
diff --git a/weed/server/volume_grpc_copy.go b/weed/server/volume_grpc_copy.go
index 7b681aa53..e5a3d6edf 100644
--- a/weed/server/volume_grpc_copy.go
+++ b/weed/server/volume_grpc_copy.go
@@ -16,6 +16,8 @@ import (
"github.com/chrislusf/seaweedfs/weed/util"
)
+const BufferSizeLimit = 1024 * 1024 * 2
+
// VolumeCopy copy the .idx .dat files, and mount the volume
func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.VolumeCopyRequest) (*volume_server_pb.VolumeCopyResponse, error) {
@@ -190,7 +192,6 @@ func (vs *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream v
bytesToRead := int64(req.StopOffset)
- const BufferSize = 1024 * 1024 * 2
var fileName = v.FileName() + req.Ext
file, err := os.Open(fileName)
if err != nil {
@@ -198,7 +199,7 @@ func (vs *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream v
}
defer file.Close()
- buffer := make([]byte, BufferSize)
+ buffer := make([]byte, BufferSizeLimit)
for bytesToRead > 0 {
bytesread, err := file.Read(buffer)
diff --git a/weed/server/volume_grpc_erasure_coding.go b/weed/server/volume_grpc_erasure_coding.go
index f82b07e29..aa0f80442 100644
--- a/weed/server/volume_grpc_erasure_coding.go
+++ b/weed/server/volume_grpc_erasure_coding.go
@@ -3,6 +3,7 @@ package weed_server
import (
"context"
"fmt"
+ "io"
"math"
"os"
@@ -71,8 +72,8 @@ func (vs *VolumeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_serv
}
// copy ec data slices
- for _, ecIndex := range req.EcIndexes {
- if err := vs.doCopyFile(ctx, client, req.VolumeId, math.MaxUint32, math.MaxInt64, baseFileName, erasure_coding.ToExt(int(ecIndex))); err != nil {
+ for _, shardId := range req.ShardIds {
+ if err := vs.doCopyFile(ctx, client, req.VolumeId, math.MaxUint32, math.MaxInt64, baseFileName, erasure_coding.ToExt(int(shardId))); err != nil {
return err
}
}
@@ -95,8 +96,8 @@ func (vs *VolumeServer) VolumeEcShardsDelete(ctx context.Context, req *volume_se
}
baseFileName := v.FileName()
- for _, shardIndex := range req.EcIndexes {
- if err := os.Remove(baseFileName + erasure_coding.ToExt(int(shardIndex))); err != nil {
+ for _, shardId := range req.ShardIds {
+ if err := os.Remove(baseFileName + erasure_coding.ToExt(int(shardId))); err != nil {
return nil, err
}
}
@@ -112,7 +113,7 @@ func (vs *VolumeServer) VolumeEcShardsDelete(ctx context.Context, req *volume_se
func (vs *VolumeServer) VolumeEcShardsMount(ctx context.Context, req *volume_server_pb.VolumeEcShardsMountRequest) (*volume_server_pb.VolumeEcShardsMountResponse, error) {
- for _, shardId := range req.EcIndexes {
+ for _, shardId := range req.ShardIds {
err := vs.store.MountEcShards(req.Collection, needle.VolumeId(req.VolumeId), erasure_coding.ShardId(shardId))
if err != nil {
@@ -131,7 +132,7 @@ func (vs *VolumeServer) VolumeEcShardsMount(ctx context.Context, req *volume_ser
func (vs *VolumeServer) VolumeEcShardsUnmount(ctx context.Context, req *volume_server_pb.VolumeEcShardsUnmountRequest) (*volume_server_pb.VolumeEcShardsUnmountResponse, error) {
- for _, shardId := range req.EcIndexes {
+ for _, shardId := range req.ShardIds {
err := vs.store.UnmountEcShards(needle.VolumeId(req.VolumeId), erasure_coding.ShardId(shardId))
if err != nil {
@@ -147,3 +148,49 @@ func (vs *VolumeServer) VolumeEcShardsUnmount(ctx context.Context, req *volume_s
return &volume_server_pb.VolumeEcShardsUnmountResponse{}, nil
}
+
+func (vs *VolumeServer) VolumeEcShardRead(req *volume_server_pb.VolumeEcShardReadRequest, stream volume_server_pb.VolumeServer_VolumeEcShardReadServer) error {
+
+ ecShards, found := vs.store.HasEcShard(needle.VolumeId(req.VolumeId))
+ if !found {
+ return fmt.Errorf("not found ec volume id %d", req.VolumeId)
+ }
+ ecShard, found := ecShards.FindEcVolumeShard(erasure_coding.ShardId(req.ShardId))
+ if !found {
+ return fmt.Errorf("not found ec shard %d.%d", req.VolumeId, req.ShardId)
+ }
+
+ buffer := make([]byte, BufferSizeLimit)
+ startOffset, bytesToRead := req.Offset, req.Size
+
+ for bytesToRead > 0 {
+ bytesread, err := ecShard.ReadAt(buffer, startOffset)
+
+ // println(fileName, "read", bytesread, "bytes, with target", bytesToRead)
+
+ if err != nil {
+ if err != io.EOF {
+ return err
+ }
+ // println(fileName, "read", bytesread, "bytes, with target", bytesToRead, "err", err.Error())
+ break
+ }
+
+ if int64(bytesread) > bytesToRead {
+ bytesread = int(bytesToRead)
+ }
+ err = stream.Send(&volume_server_pb.VolumeEcShardReadResponse{
+ Data: buffer[:bytesread],
+ })
+ if err != nil {
+ // println("sending", bytesread, "bytes err", err.Error())
+ return err
+ }
+
+ bytesToRead -= int64(bytesread)
+
+ }
+
+ return nil
+
+}
diff --git a/weed/server/volume_grpc_tail.go b/weed/server/volume_grpc_tail.go
index 667131e9f..698bad5b8 100644
--- a/weed/server/volume_grpc_tail.go
+++ b/weed/server/volume_grpc_tail.go
@@ -69,12 +69,11 @@ func sendNeedlesSince(stream volume_server_pb.VolumeServer_VolumeTailSenderServe
err = storage.ScanVolumeFileNeedleFrom(v.Version(), v.DataFile(), foundOffset.ToAcutalOffset(), func(needleHeader, needleBody []byte, needleAppendAtNs uint64) error {
- blockSizeLimit := 1024 * 1024 * 2
isLastChunk := false
// need to send body by chunks
- for i := 0; i < len(needleBody); i += blockSizeLimit {
- stopOffset := i + blockSizeLimit
+ for i := 0; i < len(needleBody); i += BufferSizeLimit {
+ stopOffset := i + BufferSizeLimit
if stopOffset >= len(needleBody) {
isLastChunk = true
stopOffset = len(needleBody)
diff --git a/weed/shell/command_ec_encode.go b/weed/shell/command_ec_encode.go
index e0d028392..499c8a32e 100644
--- a/weed/shell/command_ec_encode.go
+++ b/weed/shell/command_ec_encode.go
@@ -208,7 +208,7 @@ func oneServerCopyEcShardsFromSource(ctx context.Context, grpcDialOption grpc.Di
_, copyErr := volumeServerClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
VolumeId: uint32(volumeId),
Collection: collection,
- EcIndexes: copiedShardIds,
+ ShardIds: copiedShardIds,
SourceDataNode: existingLocation.Url,
})
if copyErr != nil {
@@ -219,7 +219,7 @@ func oneServerCopyEcShardsFromSource(ctx context.Context, grpcDialOption grpc.Di
_, mountErr := volumeServerClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
VolumeId: uint32(volumeId),
Collection: collection,
- EcIndexes: copiedShardIds,
+ ShardIds: copiedShardIds,
})
if mountErr != nil {
return mountErr
@@ -243,7 +243,7 @@ func sourceServerDeleteEcShards(ctx context.Context, grpcDialOption grpc.DialOpt
return operation.WithVolumeServerClient(sourceLocation.Url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
_, deleteErr := volumeServerClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
VolumeId: uint32(volumeId),
- EcIndexes: toBeDeletedShardIds,
+ ShardIds: toBeDeletedShardIds,
ShouldDeleteEcx: shouldDeleteEcx,
})
return deleteErr
diff --git a/weed/storage/erasure_coding/ec_locate.go b/weed/storage/erasure_coding/ec_locate.go
index ee8af3382..562966f8f 100644
--- a/weed/storage/erasure_coding/ec_locate.go
+++ b/weed/storage/erasure_coding/ec_locate.go
@@ -69,3 +69,15 @@ func locateOffsetWithinBlocks(blockLength int64, offset int64) (blockIndex int,
innerBlockOffset = offset % blockLength
return
}
+
+func (interval Interval) ToShardIdAndOffset(largeBlockSize, smallBlockSize int64) (ShardId, int64) {
+ ecFileOffset := interval.InnerBlockOffset
+ rowIndex := interval.BlockIndex / DataShardsCount
+ if interval.IsLargeBlock {
+ ecFileOffset += int64(rowIndex) * largeBlockSize
+ } else {
+ ecFileOffset += int64(interval.LargeBlockRowsCount)*largeBlockSize + int64(rowIndex)*smallBlockSize
+ }
+ ecFileIndex := interval.BlockIndex % DataShardsCount
+ return ShardId(ecFileIndex), ecFileOffset
+}
diff --git a/weed/storage/erasure_coding/ec_shard.go b/weed/storage/erasure_coding/ec_shard.go
index 084d3f03b..fa4bfcecd 100644
--- a/weed/storage/erasure_coding/ec_shard.go
+++ b/weed/storage/erasure_coding/ec_shard.go
@@ -105,3 +105,9 @@ func (shard *EcVolumeShard) findNeedleFromEcx(needleId types.NeedleId) (offset t
err = fmt.Errorf("needle id %d not found", needleId)
return
}
+
+func (shard *EcVolumeShard) ReadAt(buf []byte, offset int64) (int, error) {
+
+ return shard.ecdFile.ReadAt(buf, offset)
+
+}
diff --git a/weed/storage/erasure_coding/ec_test.go b/weed/storage/erasure_coding/ec_test.go
index 83b0bc23a..e2e872dbe 100644
--- a/weed/storage/erasure_coding/ec_test.go
+++ b/weed/storage/erasure_coding/ec_test.go
@@ -105,10 +105,8 @@ func readEcFile(datSize int64, ecFiles []*os.File, offset types.Offset, size uin
intervals := LocateData(largeBlockSize, smallBlockSize, datSize, offset.ToAcutalOffset(), size)
- nLargeBlockRows := int(datSize / (largeBlockSize * DataShardsCount))
-
for i, interval := range intervals {
- if d, e := readOneInterval(interval, ecFiles, nLargeBlockRows); e != nil {
+ if d, e := readOneInterval(interval, ecFiles); e != nil {
return nil, e
} else {
if i == 0 {
@@ -122,21 +120,14 @@ func readEcFile(datSize int64, ecFiles []*os.File, offset types.Offset, size uin
return data, nil
}
-func readOneInterval(interval Interval, ecFiles []*os.File, nLargeBlockRows int) (data []byte, err error) {
- ecFileOffset := interval.InnerBlockOffset
- rowIndex := interval.BlockIndex / DataShardsCount
- if interval.IsLargeBlock {
- ecFileOffset += int64(rowIndex) * largeBlockSize
- } else {
- ecFileOffset += int64(nLargeBlockRows)*largeBlockSize + int64(rowIndex)*smallBlockSize
- }
+func readOneInterval(interval Interval, ecFiles []*os.File) (data []byte, err error) {
- ecFileIndex := interval.BlockIndex % DataShardsCount
+ ecFileOffset, ecFileIndex := interval.ToShardIdAndOffset(largeBlockSize, smallBlockSize)
data = make([]byte, interval.Size)
err = readFromFile(ecFiles[ecFileIndex], data, ecFileOffset)
{ // do some ec testing
- ecData, err := readFromOtherEcFiles(ecFiles, ecFileIndex, ecFileOffset, interval.Size)
+ ecData, err := readFromOtherEcFiles(ecFiles, int(ecFileIndex), ecFileOffset, interval.Size)
if err != nil {
return nil, fmt.Errorf("ec reconstruct error: %v", err)
}
@@ -198,7 +189,7 @@ func TestLocateData(t *testing.T) {
if len(intervals) != 1 {
t.Errorf("unexpected interval size %d", len(intervals))
}
- if !intervals[0].sameAs(Interval{0, 0, 1, false}) {
+ if !intervals[0].sameAs(Interval{0, 0, 1, false, 1}) {
t.Errorf("unexpected interval %+v", intervals[0])
}
diff --git a/weed/storage/erasure_coding/ec_volume.go b/weed/storage/erasure_coding/ec_volume.go
index d57a28449..881e88d5f 100644
--- a/weed/storage/erasure_coding/ec_volume.go
+++ b/weed/storage/erasure_coding/ec_volume.go
@@ -6,6 +6,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
+ "github.com/chrislusf/seaweedfs/weed/storage/types"
)
type EcVolumeShards []*EcVolumeShard
@@ -71,19 +72,17 @@ func (shards *EcVolumeShards) ToVolumeEcShardInformationMessage() (messages []*m
return
}
-func (shards *EcVolumeShards) ReadEcShardNeedle(n *needle.Needle) (int, error) {
+func (shards *EcVolumeShards) LocateEcShardNeedle(n *needle.Needle) (offset types.Offset, size uint32, intervals []Interval, err error) {
shard := (*shards)[0]
// find the needle from ecx file
- offset, size, err := shard.findNeedleFromEcx(n.Id)
+ offset, size, err = shard.findNeedleFromEcx(n.Id)
if err != nil {
- return 0, err
+ return types.Offset{}, 0, nil, err
}
// calculate the locations in the ec shards
- intervals := LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, shard.ecxFileSize, offset.ToAcutalOffset(), size)
+ intervals = LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, shard.ecxFileSize, offset.ToAcutalOffset(), size)
- // TODO read the intervals
-
- return len(intervals), nil
+ return
}
diff --git a/weed/storage/needle/needle_read_write.go b/weed/storage/needle/needle_read_write.go
index 85f335892..f2d699d11 100644
--- a/weed/storage/needle/needle_read_write.go
+++ b/weed/storage/needle/needle_read_write.go
@@ -170,11 +170,8 @@ func ReadNeedleBlob(r *os.File, offset int64, size uint32, version Version) (dat
return dataSlice, err
}
-func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version) (err error) {
- bytes, err := ReadNeedleBlob(r, offset, size, version)
- if err != nil {
- return err
- }
+// ReadBytes hydrates the needle from the bytes buffer, with only n.Id is set.
+func (n *Needle) ReadBytes(bytes []byte, offset int64, size uint32, version Version) (err error) {
n.ParseNeedleHeader(bytes)
if n.Size != size {
return fmt.Errorf("File Entry Not Found. offset %d, Needle id %d expected size %d Memory %d", offset, n.Id, n.Size, size)
@@ -203,6 +200,15 @@ func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version
return nil
}
+// ReadData hydrates the needle from the file, with only n.Id is set.
+func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version) (err error) {
+ bytes, err := ReadNeedleBlob(r, offset, size, version)
+ if err != nil {
+ return err
+ }
+ return n.ReadBytes(bytes, offset, size, version)
+}
+
func (n *Needle) ParseNeedleHeader(bytes []byte) {
n.Cookie = BytesToCookie(bytes[0:CookieSize])
n.Id = BytesToNeedleId(bytes[CookieSize : CookieSize+NeedleIdSize])
diff --git a/weed/storage/store_ec.go b/weed/storage/store_ec.go
index ed7c6484b..7514339c9 100644
--- a/weed/storage/store_ec.go
+++ b/weed/storage/store_ec.go
@@ -89,9 +89,62 @@ func (s *Store) HasEcShard(vid needle.VolumeId) (erasure_coding.EcVolumeShards,
func (s *Store) ReadEcShardNeedle(vid needle.VolumeId, n *needle.Needle) (int, error) {
for _, location := range s.Locations {
- if ecShards, found := location.HasEcShard(vid); found {
- return ecShards.ReadEcShardNeedle(n)
+ if localEcShards, found := location.HasEcShard(vid); found {
+
+ offset, size, intervals, err := localEcShards.LocateEcShardNeedle(n)
+ if err != nil {
+ return 0, err
+ }
+
+ bytes, err := s.ReadEcShardIntervals(vid, localEcShards, intervals)
+ if err != nil {
+ return 0, fmt.Errorf("ReadEcShardIntervals: %v", err)
+ }
+
+ version := needle.CurrentVersion
+
+ err = n.ReadBytes(bytes, offset.ToAcutalOffset(), size, version)
+ if err != nil {
+ return 0, fmt.Errorf("readbytes: %v", err)
+ }
+
+ return len(bytes), nil
}
}
return 0, fmt.Errorf("ec shard %d not found", vid)
}
+
+func (s *Store) ReadEcShardIntervals(vid needle.VolumeId, localEcShards erasure_coding.EcVolumeShards, intervals []erasure_coding.Interval) (data []byte, err error) {
+ for i, interval := range intervals {
+ if d, e := s.readOneEcShardInterval(vid, localEcShards, interval); e != nil {
+ return nil, e
+ } else {
+ if i == 0 {
+ data = d
+ } else {
+ data = append(data, d...)
+ }
+ }
+ }
+ return
+}
+
+func (s *Store) readOneEcShardInterval(vid needle.VolumeId, localEcShards erasure_coding.EcVolumeShards, interval erasure_coding.Interval) (data []byte, err error) {
+ shardId, actualOffset := interval.ToShardIdAndOffset(erasure_coding.ErasureCodingLargeBlockSize, erasure_coding.ErasureCodingSmallBlockSize)
+ data = make([]byte, interval.Size)
+ if shard, found := localEcShards.FindEcVolumeShard(shardId); found {
+ if _, err = shard.ReadAt(data, actualOffset); err != nil {
+ return
+ }
+ } else {
+ s.readOneRemoteEcShardInterval(vid, shardId, data, actualOffset)
+ }
+ return
+}
+
+func (s *Store) readOneRemoteEcShardInterval(vid needle.VolumeId, shardId erasure_coding.ShardId, buf []byte, offset int64) (n int, err error) {
+
+
+
+ return
+}
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-21 08:03:09 +0000