aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--unmaintained/see_idx/see_idx.go4
-rw-r--r--weed/command/server.go2
-rw-r--r--weed/pb/master.proto13
-rw-r--r--weed/pb/master_pb/master.pb.go304
-rw-r--r--weed/server/master_grpc_server.go4
-rw-r--r--weed/server/volume_grpc_client_to_master.go38
-rw-r--r--weed/storage/disk_location.go69
-rw-r--r--weed/storage/disk_location_ec.go84
-rw-r--r--weed/storage/disk_location_ec_test.go17
-rw-r--r--weed/storage/erasure_coding/3.ec070
-rw-r--r--weed/storage/erasure_coding/3.ecx0
-rw-r--r--weed/storage/erasure_coding/ec_encoder.go4
-rw-r--r--weed/storage/erasure_coding/ec_volume.go108
-rw-r--r--weed/storage/idx/walk.go54
-rw-r--r--weed/storage/needle_map.go7
-rw-r--r--weed/storage/needle_map_leveldb.go6
-rw-r--r--weed/storage/needle_map_memory.go40
-rw-r--r--weed/storage/needle_map_metric.go3
-rw-r--r--weed/storage/store.go7
-rw-r--r--weed/storage/store_ec.go21
-rw-r--r--weed/storage/volume_backup.go5
-rw-r--r--weed/storage/volume_checking.go3
-rw-r--r--weed/storage/volume_vacuum.go5
23 files changed, 591 insertions, 207 deletions
diff --git a/unmaintained/see_idx/see_idx.go b/unmaintained/see_idx/see_idx.go
index 23ca04c2e..777af1821 100644
--- a/unmaintained/see_idx/see_idx.go
+++ b/unmaintained/see_idx/see_idx.go
@@ -8,7 +8,7 @@ import (
"strconv"
"github.com/chrislusf/seaweedfs/weed/glog"
- "github.com/chrislusf/seaweedfs/weed/storage"
+ "github.com/chrislusf/seaweedfs/weed/storage/idx"
"github.com/chrislusf/seaweedfs/weed/storage/types"
)
@@ -35,7 +35,7 @@ func main() {
}
defer indexFile.Close()
- storage.WalkIndexFile(indexFile, func(key types.NeedleId, offset types.Offset, size uint32) error {
+ idx.WalkIndexFile(indexFile, func(key types.NeedleId, offset types.Offset, size uint32) error {
fmt.Printf("key:%v offset:%v size:%v\n", key, offset, size)
return nil
})
diff --git a/weed/command/server.go b/weed/command/server.go
index e015e8832..630ba72a7 100644
--- a/weed/command/server.go
+++ b/weed/command/server.go
@@ -222,7 +222,7 @@ func runServer(cmd *Command, args []string) bool {
glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
}
// Create your protocol servers.
- glog.V(0).Infof("grpc config %+v", viper.Sub("grpc"))
+ glog.V(1).Infof("grpc config %+v", viper.Sub("grpc"))
grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "master"))
master_pb.RegisterSeaweedServer(grpcS, ms)
protobuf.RegisterRaftServer(grpcS, raftServer)
diff --git a/weed/pb/master.proto b/weed/pb/master.proto
index e6d34b826..0968d2dcf 100644
--- a/weed/pb/master.proto
+++ b/weed/pb/master.proto
@@ -38,6 +38,13 @@ message Heartbeat {
// delta volumes
repeated VolumeShortInformationMessage new_volumes = 10;
repeated VolumeShortInformationMessage deleted_volumes = 11;
+
+ // erasure coding
+ repeated VolumeEcShardInformationMessage ec_shards = 16;
+ // delta erasure coding shards
+ repeated VolumeEcShardInformationMessage new_ec_shards = 17;
+ repeated VolumeEcShardInformationMessage deleted_ec_shards = 18;
+
}
message HeartbeatResponse {
@@ -67,6 +74,12 @@ message VolumeShortInformationMessage {
uint32 ttl = 10;
}
+message VolumeEcShardInformationMessage {
+ uint32 id = 1;
+ string collection = 2;
+ uint32 ec_index = 3;
+}
+
message Empty {
}
diff --git a/weed/pb/master_pb/master.pb.go b/weed/pb/master_pb/master.pb.go
index 4c2e893a6..0c7a9837e 100644
--- a/weed/pb/master_pb/master.pb.go
+++ b/weed/pb/master_pb/master.pb.go
@@ -13,6 +13,7 @@ It has these top-level messages:
HeartbeatResponse
VolumeInformationMessage
VolumeShortInformationMessage
+ VolumeEcShardInformationMessage
Empty
SuperBlockExtra
ClientListenRequest
@@ -72,6 +73,11 @@ type Heartbeat struct {
// delta volumes
NewVolumes []*VolumeShortInformationMessage `protobuf:"bytes,10,rep,name=new_volumes,json=newVolumes" json:"new_volumes,omitempty"`
DeletedVolumes []*VolumeShortInformationMessage `protobuf:"bytes,11,rep,name=deleted_volumes,json=deletedVolumes" json:"deleted_volumes,omitempty"`
+ // erasure coding
+ EcShards []*VolumeEcShardInformationMessage `protobuf:"bytes,16,rep,name=ec_shards,json=ecShards" json:"ec_shards,omitempty"`
+ // delta erasure coding shards
+ NewEcShards []*VolumeEcShardInformationMessage `protobuf:"bytes,17,rep,name=new_ec_shards,json=newEcShards" json:"new_ec_shards,omitempty"`
+ DeletedEcShards []*VolumeEcShardInformationMessage `protobuf:"bytes,18,rep,name=deleted_ec_shards,json=deletedEcShards" json:"deleted_ec_shards,omitempty"`
}
func (m *Heartbeat) Reset() { *m = Heartbeat{} }
@@ -156,6 +162,27 @@ func (m *Heartbeat) GetDeletedVolumes() []*VolumeShortInformationMessage {
return nil
}
+func (m *Heartbeat) GetEcShards() []*VolumeEcShardInformationMessage {
+ if m != nil {
+ return m.EcShards
+ }
+ return nil
+}
+
+func (m *Heartbeat) GetNewEcShards() []*VolumeEcShardInformationMessage {
+ if m != nil {
+ return m.NewEcShards
+ }
+ return nil
+}
+
+func (m *Heartbeat) GetDeletedEcShards() []*VolumeEcShardInformationMessage {
+ if m != nil {
+ return m.DeletedEcShards
+ }
+ return nil
+}
+
type HeartbeatResponse struct {
VolumeSizeLimit uint64 `protobuf:"varint,1,opt,name=volumeSizeLimit" json:"volumeSizeLimit,omitempty"`
Leader string `protobuf:"bytes,3,opt,name=leader" json:"leader,omitempty"`
@@ -324,13 +351,45 @@ func (m *VolumeShortInformationMessage) GetTtl() uint32 {
return 0
}
+type VolumeEcShardInformationMessage struct {
+ Id uint32 `protobuf:"varint,1,opt,name=id" json:"id,omitempty"`
+ Collection string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"`
+ EcIndex uint32 `protobuf:"varint,3,opt,name=ec_index,json=ecIndex" json:"ec_index,omitempty"`
+}
+
+func (m *VolumeEcShardInformationMessage) Reset() { *m = VolumeEcShardInformationMessage{} }
+func (m *VolumeEcShardInformationMessage) String() string { return proto.CompactTextString(m) }
+func (*VolumeEcShardInformationMessage) ProtoMessage() {}
+func (*VolumeEcShardInformationMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
+
+func (m *VolumeEcShardInformationMessage) GetId() uint32 {
+ if m != nil {
+ return m.Id
+ }
+ return 0
+}
+
+func (m *VolumeEcShardInformationMessage) GetCollection() string {
+ if m != nil {
+ return m.Collection
+ }
+ return ""
+}
+
+func (m *VolumeEcShardInformationMessage) GetEcIndex() uint32 {
+ if m != nil {
+ return m.EcIndex
+ }
+ return 0
+}
+
type Empty struct {
}
func (m *Empty) Reset() { *m = Empty{} }
func (m *Empty) String() string { return proto.CompactTextString(m) }
func (*Empty) ProtoMessage() {}
-func (*Empty) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
+func (*Empty) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
type SuperBlockExtra struct {
ErasureCoding *SuperBlockExtra_ErasureCoding `protobuf:"bytes,1,opt,name=erasure_coding,json=erasureCoding" json:"erasure_coding,omitempty"`
@@ -339,7 +398,7 @@ type SuperBlockExtra struct {
func (m *SuperBlockExtra) Reset() { *m = SuperBlockExtra{} }
func (m *SuperBlockExtra) String() string { return proto.CompactTextString(m) }
func (*SuperBlockExtra) ProtoMessage() {}
-func (*SuperBlockExtra) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
+func (*SuperBlockExtra) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
func (m *SuperBlockExtra) GetErasureCoding() *SuperBlockExtra_ErasureCoding {
if m != nil {
@@ -358,7 +417,7 @@ func (m *SuperBlockExtra_ErasureCoding) Reset() { *m = SuperBlockExtra_E
func (m *SuperBlockExtra_ErasureCoding) String() string { return proto.CompactTextString(m) }
func (*SuperBlockExtra_ErasureCoding) ProtoMessage() {}
func (*SuperBlockExtra_ErasureCoding) Descriptor() ([]byte, []int) {
- return fileDescriptor0, []int{5, 0}
+ return fileDescriptor0, []int{6, 0}
}
func (m *SuperBlockExtra_ErasureCoding) GetData() uint32 {
@@ -389,7 +448,7 @@ type ClientListenRequest struct {
func (m *ClientListenRequest) Reset() { *m = ClientListenRequest{} }
func (m *ClientListenRequest) String() string { return proto.CompactTextString(m) }
func (*ClientListenRequest) ProtoMessage() {}
-func (*ClientListenRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
+func (*ClientListenRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
func (m *ClientListenRequest) GetName() string {
if m != nil {
@@ -408,7 +467,7 @@ type VolumeLocation struct {
func (m *VolumeLocation) Reset() { *m = VolumeLocation{} }
func (m *VolumeLocation) String() string { return proto.CompactTextString(m) }
func (*VolumeLocation) ProtoMessage() {}
-func (*VolumeLocation) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
+func (*VolumeLocation) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
func (m *VolumeLocation) GetUrl() string {
if m != nil {
@@ -446,7 +505,7 @@ type LookupVolumeRequest struct {
func (m *LookupVolumeRequest) Reset() { *m = LookupVolumeRequest{} }
func (m *LookupVolumeRequest) String() string { return proto.CompactTextString(m) }
func (*LookupVolumeRequest) ProtoMessage() {}
-func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
+func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
func (m *LookupVolumeRequest) GetVolumeIds() []string {
if m != nil {
@@ -469,7 +528,7 @@ type LookupVolumeResponse struct {
func (m *LookupVolumeResponse) Reset() { *m = LookupVolumeResponse{} }
func (m *LookupVolumeResponse) String() string { return proto.CompactTextString(m) }
func (*LookupVolumeResponse) ProtoMessage() {}
-func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
+func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
func (m *LookupVolumeResponse) GetVolumeIdLocations() []*LookupVolumeResponse_VolumeIdLocation {
if m != nil {
@@ -488,7 +547,7 @@ func (m *LookupVolumeResponse_VolumeIdLocation) Reset() { *m = LookupVol
func (m *LookupVolumeResponse_VolumeIdLocation) String() string { return proto.CompactTextString(m) }
func (*LookupVolumeResponse_VolumeIdLocation) ProtoMessage() {}
func (*LookupVolumeResponse_VolumeIdLocation) Descriptor() ([]byte, []int) {
- return fileDescriptor0, []int{9, 0}
+ return fileDescriptor0, []int{10, 0}
}
func (m *LookupVolumeResponse_VolumeIdLocation) GetVolumeId() string {
@@ -520,7 +579,7 @@ type Location struct {
func (m *Location) Reset() { *m = Location{} }
func (m *Location) String() string { return proto.CompactTextString(m) }
func (*Location) ProtoMessage() {}
-func (*Location) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
+func (*Location) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
func (m *Location) GetUrl() string {
if m != nil {
@@ -549,7 +608,7 @@ type AssignRequest struct {
func (m *AssignRequest) Reset() { *m = AssignRequest{} }
func (m *AssignRequest) String() string { return proto.CompactTextString(m) }
func (*AssignRequest) ProtoMessage() {}
-func (*AssignRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
+func (*AssignRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} }
func (m *AssignRequest) GetCount() uint64 {
if m != nil {
@@ -612,7 +671,7 @@ type AssignResponse struct {
func (m *AssignResponse) Reset() { *m = AssignResponse{} }
func (m *AssignResponse) String() string { return proto.CompactTextString(m) }
func (*AssignResponse) ProtoMessage() {}
-func (*AssignResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} }
+func (*AssignResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} }
func (m *AssignResponse) GetFid() string {
if m != nil {
@@ -665,7 +724,7 @@ type StatisticsRequest struct {
func (m *StatisticsRequest) Reset() { *m = StatisticsRequest{} }
func (m *StatisticsRequest) String() string { return proto.CompactTextString(m) }
func (*StatisticsRequest) ProtoMessage() {}
-func (*StatisticsRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} }
+func (*StatisticsRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} }
func (m *StatisticsRequest) GetReplication() string {
if m != nil {
@@ -700,7 +759,7 @@ type StatisticsResponse struct {
func (m *StatisticsResponse) Reset() { *m = StatisticsResponse{} }
func (m *StatisticsResponse) String() string { return proto.CompactTextString(m) }
func (*StatisticsResponse) ProtoMessage() {}
-func (*StatisticsResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} }
+func (*StatisticsResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
func (m *StatisticsResponse) GetReplication() string {
if m != nil {
@@ -752,7 +811,7 @@ type StorageType struct {
func (m *StorageType) Reset() { *m = StorageType{} }
func (m *StorageType) String() string { return proto.CompactTextString(m) }
func (*StorageType) ProtoMessage() {}
-func (*StorageType) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
+func (*StorageType) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} }
func (m *StorageType) GetReplication() string {
if m != nil {
@@ -775,7 +834,7 @@ type Collection struct {
func (m *Collection) Reset() { *m = Collection{} }
func (m *Collection) String() string { return proto.CompactTextString(m) }
func (*Collection) ProtoMessage() {}
-func (*Collection) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} }
+func (*Collection) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
func (m *Collection) GetName() string {
if m != nil {
@@ -790,7 +849,7 @@ type CollectionListRequest struct {
func (m *CollectionListRequest) Reset() { *m = CollectionListRequest{} }
func (m *CollectionListRequest) String() string { return proto.CompactTextString(m) }
func (*CollectionListRequest) ProtoMessage() {}
-func (*CollectionListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
+func (*CollectionListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
type CollectionListResponse struct {
Collections []*Collection `protobuf:"bytes,1,rep,name=collections" json:"collections,omitempty"`
@@ -799,7 +858,7 @@ type CollectionListResponse struct {
func (m *CollectionListResponse) Reset() { *m = CollectionListResponse{} }
func (m *CollectionListResponse) String() string { return proto.CompactTextString(m) }
func (*CollectionListResponse) ProtoMessage() {}
-func (*CollectionListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
+func (*CollectionListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
func (m *CollectionListResponse) GetCollections() []*Collection {
if m != nil {
@@ -815,7 +874,7 @@ type CollectionDeleteRequest struct {
func (m *CollectionDeleteRequest) Reset() { *m = CollectionDeleteRequest{} }
func (m *CollectionDeleteRequest) String() string { return proto.CompactTextString(m) }
func (*CollectionDeleteRequest) ProtoMessage() {}
-func (*CollectionDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
+func (*CollectionDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
func (m *CollectionDeleteRequest) GetName() string {
if m != nil {
@@ -830,7 +889,7 @@ type CollectionDeleteResponse struct {
func (m *CollectionDeleteResponse) Reset() { *m = CollectionDeleteResponse{} }
func (m *CollectionDeleteResponse) String() string { return proto.CompactTextString(m) }
func (*CollectionDeleteResponse) ProtoMessage() {}
-func (*CollectionDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
+func (*CollectionDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
//
// volume related
@@ -847,7 +906,7 @@ type DataNodeInfo struct {
func (m *DataNodeInfo) Reset() { *m = DataNodeInfo{} }
func (m *DataNodeInfo) String() string { return proto.CompactTextString(m) }
func (*DataNodeInfo) ProtoMessage() {}
-func (*DataNodeInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
+func (*DataNodeInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
func (m *DataNodeInfo) GetId() string {
if m != nil {
@@ -903,7 +962,7 @@ type RackInfo struct {
func (m *RackInfo) Reset() { *m = RackInfo{} }
func (m *RackInfo) String() string { return proto.CompactTextString(m) }
func (*RackInfo) ProtoMessage() {}
-func (*RackInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
+func (*RackInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
func (m *RackInfo) GetId() string {
if m != nil {
@@ -959,7 +1018,7 @@ type DataCenterInfo struct {
func (m *DataCenterInfo) Reset() { *m = DataCenterInfo{} }
func (m *DataCenterInfo) String() string { return proto.CompactTextString(m) }
func (*DataCenterInfo) ProtoMessage() {}
-func (*DataCenterInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
+func (*DataCenterInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} }
func (m *DataCenterInfo) GetId() string {
if m != nil {
@@ -1015,7 +1074,7 @@ type TopologyInfo struct {
func (m *TopologyInfo) Reset() { *m = TopologyInfo{} }
func (m *TopologyInfo) String() string { return proto.CompactTextString(m) }
func (*TopologyInfo) ProtoMessage() {}
-func (*TopologyInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} }
+func (*TopologyInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
func (m *TopologyInfo) GetId() string {
if m != nil {
@@ -1065,7 +1124,7 @@ type VolumeListRequest struct {
func (m *VolumeListRequest) Reset() { *m = VolumeListRequest{} }
func (m *VolumeListRequest) String() string { return proto.CompactTextString(m) }
func (*VolumeListRequest) ProtoMessage() {}
-func (*VolumeListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
+func (*VolumeListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} }
type VolumeListResponse struct {
TopologyInfo *TopologyInfo `protobuf:"bytes,1,opt,name=topology_info,json=topologyInfo" json:"topology_info,omitempty"`
@@ -1075,7 +1134,7 @@ type VolumeListResponse struct {
func (m *VolumeListResponse) Reset() { *m = VolumeListResponse{} }
func (m *VolumeListResponse) String() string { return proto.CompactTextString(m) }
func (*VolumeListResponse) ProtoMessage() {}
-func (*VolumeListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} }
+func (*VolumeListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27} }
func (m *VolumeListResponse) GetTopologyInfo() *TopologyInfo {
if m != nil {
@@ -1096,6 +1155,7 @@ func init() {
proto.RegisterType((*HeartbeatResponse)(nil), "master_pb.HeartbeatResponse")
proto.RegisterType((*VolumeInformationMessage)(nil), "master_pb.VolumeInformationMessage")
proto.RegisterType((*VolumeShortInformationMessage)(nil), "master_pb.VolumeShortInformationMessage")
+ proto.RegisterType((*VolumeEcShardInformationMessage)(nil), "master_pb.VolumeEcShardInformationMessage")
proto.RegisterType((*Empty)(nil), "master_pb.Empty")
proto.RegisterType((*SuperBlockExtra)(nil), "master_pb.SuperBlockExtra")
proto.RegisterType((*SuperBlockExtra_ErasureCoding)(nil), "master_pb.SuperBlockExtra.ErasureCoding")
@@ -1494,98 +1554,104 @@ var _Seaweed_serviceDesc = grpc.ServiceDesc{
func init() { proto.RegisterFile("master.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
- // 1482 bytes of a gzipped FileDescriptorProto
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xd4, 0x58, 0x4d, 0x6f, 0xdb, 0x46,
- 0x13, 0x36, 0xa9, 0x0f, 0x4b, 0xa3, 0xef, 0xb5, 0x93, 0x30, 0xca, 0x9b, 0x44, 0x61, 0x2e, 0xca,
- 0xfb, 0xe1, 0x37, 0x75, 0x0f, 0x3d, 0xb4, 0x45, 0x90, 0x38, 0x0e, 0x6a, 0xc4, 0x69, 0x12, 0x2a,
- 0x49, 0x81, 0x02, 0x05, 0xbb, 0x22, 0xd7, 0x0e, 0x61, 0x8a, 0x64, 0xc9, 0x95, 0x63, 0xe5, 0xd2,
- 0x43, 0x7b, 0x2c, 0xda, 0x43, 0xff, 0x44, 0x7f, 0x45, 0x2f, 0x3d, 0xe6, 0xc7, 0x14, 0xe8, 0xbd,
- 0x40, 0xb1, 0x1f, 0x24, 0x97, 0x94, 0x6c, 0xa7, 0x05, 0x72, 0xc8, 0x6d, 0x77, 0x66, 0x76, 0x76,
- 0xf6, 0x99, 0xe1, 0x33, 0x23, 0x41, 0x7b, 0x86, 0x13, 0x4a, 0xe2, 0xad, 0x28, 0x0e, 0x69, 0x88,
- 0x9a, 0x62, 0x67, 0x47, 0x53, 0xf3, 0x4d, 0x05, 0x9a, 0x9f, 0x11, 0x1c, 0xd3, 0x29, 0xc1, 0x14,
- 0x75, 0x41, 0xf7, 0x22, 0x43, 0x1b, 0x69, 0xe3, 0xa6, 0xa5, 0x7b, 0x11, 0x42, 0x50, 0x8d, 0xc2,
- 0x98, 0x1a, 0xfa, 0x48, 0x1b, 0x77, 0x2c, 0xbe, 0x46, 0x57, 0x01, 0xa2, 0xf9, 0xd4, 0xf7, 0x1c,
- 0x7b, 0x1e, 0xfb, 0x46, 0x85, 0xdb, 0x36, 0x85, 0xe4, 0x79, 0xec, 0xa3, 0x31, 0xf4, 0x67, 0xf8,
- 0xc4, 0x3e, 0x0e, 0xfd, 0xf9, 0x8c, 0xd8, 0x4e, 0x38, 0x0f, 0xa8, 0x51, 0xe5, 0xc7, 0xbb, 0x33,
- 0x7c, 0xf2, 0x82, 0x8b, 0x77, 0x98, 0x14, 0x8d, 0x58, 0x54, 0x27, 0xf6, 0x81, 0xe7, 0x13, 0xfb,
- 0x88, 0x2c, 0x8c, 0xda, 0x48, 0x1b, 0x57, 0x2d, 0x98, 0xe1, 0x93, 0x07, 0x9e, 0x4f, 0x1e, 0x92,
- 0x05, 0xba, 0x0e, 0x2d, 0x17, 0x53, 0x6c, 0x3b, 0x24, 0xa0, 0x24, 0x36, 0xea, 0xfc, 0x2e, 0x60,
- 0xa2, 0x1d, 0x2e, 0x61, 0xf1, 0xc5, 0xd8, 0x39, 0x32, 0xd6, 0xb9, 0x86, 0xaf, 0x59, 0x7c, 0xd8,
- 0x9d, 0x79, 0x81, 0xcd, 0x23, 0x6f, 0xf0, 0xab, 0x9b, 0x5c, 0xf2, 0x84, 0x85, 0xff, 0x29, 0xac,
- 0x8b, 0xd8, 0x12, 0xa3, 0x39, 0xaa, 0x8c, 0x5b, 0xdb, 0x37, 0xb7, 0x32, 0x34, 0xb6, 0x44, 0x78,
- 0x7b, 0xc1, 0x41, 0x18, 0xcf, 0x30, 0xf5, 0xc2, 0xe0, 0x11, 0x49, 0x12, 0x7c, 0x48, 0xac, 0xf4,
- 0x0c, 0xda, 0x83, 0x56, 0x40, 0x5e, 0xd9, 0xa9, 0x0b, 0xe0, 0x2e, 0xc6, 0x4b, 0x2e, 0x26, 0x2f,
- 0xc3, 0x98, 0xae, 0xf0, 0x03, 0x01, 0x79, 0xf5, 0x42, 0xba, 0x7a, 0x0a, 0x3d, 0x97, 0xf8, 0x84,
- 0x12, 0x37, 0x73, 0xd7, 0xfa, 0x9b, 0xee, 0xba, 0xd2, 0x81, 0x74, 0x69, 0x3e, 0x87, 0x41, 0x96,
- 0x4c, 0x8b, 0x24, 0x51, 0x18, 0x24, 0x04, 0x8d, 0xa1, 0x27, 0xfc, 0x4f, 0xbc, 0xd7, 0x64, 0xdf,
- 0x9b, 0x79, 0x94, 0x67, 0xb8, 0x6a, 0x95, 0xc5, 0xe8, 0x22, 0xd4, 0x7d, 0x82, 0x5d, 0x12, 0xcb,
- 0xb4, 0xca, 0x9d, 0xf9, 0xbb, 0x0e, 0xc6, 0x69, 0xd0, 0xf0, 0x9a, 0x71, 0xb9, 0xc7, 0x8e, 0xa5,
- 0x7b, 0x2e, 0xcb, 0x49, 0xe2, 0xbd, 0x26, 0xbc, 0x66, 0xaa, 0x16, 0x5f, 0xa3, 0x6b, 0x00, 0x4e,
- 0xe8, 0xfb, 0xc4, 0x61, 0x07, 0xa5, 0x73, 0x45, 0xc2, 0x72, 0xc6, 0xcb, 0x20, 0x2f, 0x97, 0xaa,
- 0xd5, 0x64, 0x12, 0x51, 0x29, 0x37, 0xa0, 0x2d, 0x1e, 0x2a, 0x0d, 0x44, 0xa5, 0xb4, 0x84, 0x4c,
- 0x98, 0xfc, 0x17, 0x50, 0x0a, 0xe6, 0x74, 0x91, 0x19, 0xd6, 0xb9, 0x61, 0x5f, 0x6a, 0xee, 0x2d,
- 0x52, 0xeb, 0x2b, 0xd0, 0x8c, 0x09, 0x76, 0xed, 0x30, 0xf0, 0x17, 0xbc, 0x78, 0x1a, 0x56, 0x83,
- 0x09, 0x1e, 0x07, 0xfe, 0x02, 0xfd, 0x07, 0x06, 0x31, 0x89, 0x7c, 0xcf, 0xc1, 0x76, 0xe4, 0x63,
- 0x87, 0xcc, 0x48, 0x90, 0xd6, 0x51, 0x5f, 0x2a, 0x9e, 0xa4, 0x72, 0x64, 0xc0, 0xfa, 0x31, 0x89,
- 0x13, 0xf6, 0xac, 0x26, 0x37, 0x49, 0xb7, 0xa8, 0x0f, 0x15, 0x4a, 0x7d, 0x03, 0xb8, 0x94, 0x2d,
- 0xd1, 0x2d, 0xe8, 0x3b, 0xe1, 0x2c, 0xc2, 0x0e, 0xb5, 0x63, 0x72, 0xec, 0xf1, 0x43, 0x2d, 0xae,
- 0xee, 0x49, 0xb9, 0x25, 0xc5, 0xe6, 0x2f, 0x1a, 0x5c, 0x3d, 0x33, 0xf5, 0x4b, 0xb0, 0x9f, 0x07,
- 0xf1, 0xbb, 0x7a, 0x95, 0xb9, 0x0e, 0xb5, 0xdd, 0x59, 0x44, 0x17, 0xe6, 0xaf, 0x1a, 0xf4, 0x26,
- 0xf3, 0x88, 0xc4, 0xf7, 0xfc, 0xd0, 0x39, 0xda, 0x3d, 0xa1, 0x31, 0x46, 0x8f, 0xa1, 0x4b, 0x62,
- 0x9c, 0xcc, 0x63, 0x96, 0x11, 0xd7, 0x0b, 0x0e, 0x79, 0xc4, 0xc5, 0x12, 0x2f, 0x9d, 0xd9, 0xda,
- 0x15, 0x07, 0x76, 0xb8, 0xbd, 0xd5, 0x21, 0xea, 0x76, 0xf8, 0x25, 0x74, 0x0a, 0x7a, 0x56, 0x6e,
- 0x8c, 0x10, 0x24, 0x12, 0x7c, 0xcd, 0xea, 0x38, 0xc2, 0xb1, 0x47, 0x17, 0x92, 0xb8, 0xe4, 0x8e,
- 0x95, 0x99, 0xe4, 0x25, 0xcf, 0x4d, 0x8c, 0xca, 0xa8, 0xc2, 0xa8, 0x41, 0x48, 0xf6, 0xdc, 0xc4,
- 0xbc, 0x05, 0x1b, 0x3b, 0xbe, 0x47, 0x02, 0xba, 0xef, 0x25, 0x94, 0x04, 0x16, 0xf9, 0x66, 0x4e,
- 0x12, 0xca, 0x6e, 0x08, 0xf0, 0x8c, 0x48, 0x5a, 0xe4, 0x6b, 0xf3, 0x5b, 0xe8, 0x8a, 0xf4, 0xec,
- 0x87, 0x0e, 0x4f, 0x0b, 0x03, 0x86, 0xf1, 0xa1, 0x30, 0x62, 0xcb, 0x12, 0x51, 0xea, 0x65, 0xa2,
- 0xbc, 0x0c, 0x0d, 0xce, 0x24, 0x79, 0x28, 0xeb, 0x8c, 0x1c, 0x3c, 0x37, 0xc9, 0xeb, 0xdd, 0x15,
- 0xea, 0x2a, 0x57, 0xb7, 0xd2, 0x8f, 0xdd, 0x73, 0x13, 0xf3, 0x19, 0x6c, 0xec, 0x87, 0xe1, 0xd1,
- 0x3c, 0x12, 0x61, 0xa4, 0xb1, 0x16, 0x5f, 0xa8, 0x8d, 0x2a, 0xec, 0xce, 0xec, 0x85, 0xa5, 0x22,
- 0xd1, 0xcb, 0x45, 0x62, 0xfe, 0xa1, 0xc1, 0x66, 0xd1, 0xad, 0xe4, 0x90, 0xaf, 0x61, 0x23, 0xf3,
- 0x6b, 0xfb, 0xf2, 0xcd, 0xe2, 0x82, 0xd6, 0xf6, 0x6d, 0x25, 0x99, 0xab, 0x4e, 0xa7, 0xb4, 0xea,
- 0xa6, 0x60, 0x59, 0x83, 0xe3, 0x92, 0x24, 0x19, 0x9e, 0x40, 0xbf, 0x6c, 0xc6, 0x3e, 0xd3, 0xec,
- 0x56, 0x89, 0x6c, 0x23, 0x3d, 0x89, 0x3e, 0x80, 0x66, 0x1e, 0x88, 0xce, 0x03, 0xd9, 0x28, 0x04,
- 0x22, 0xef, 0xca, 0xad, 0xd0, 0x26, 0xd4, 0x48, 0x1c, 0x87, 0x29, 0xbd, 0x89, 0x8d, 0xf9, 0x31,
- 0x34, 0xfe, 0x71, 0x16, 0xcd, 0x37, 0x1a, 0x74, 0xee, 0x26, 0x89, 0x77, 0x98, 0x95, 0xcb, 0x26,
- 0xd4, 0x04, 0xf9, 0x08, 0x92, 0x15, 0x1b, 0x34, 0x82, 0x96, 0xfc, 0xca, 0x14, 0xe8, 0x55, 0xd1,
- 0xb9, 0x1f, 0xb0, 0xfc, 0xf2, 0xaa, 0x22, 0x34, 0xc6, 0x27, 0xa5, 0xf6, 0x58, 0x3b, 0xb5, 0x3d,
- 0xd6, 0x95, 0xf6, 0x78, 0x05, 0x9a, 0xfc, 0x50, 0x10, 0xba, 0x44, 0xf6, 0xcd, 0x06, 0x13, 0x7c,
- 0x1e, 0xba, 0xc4, 0xfc, 0x59, 0x83, 0x6e, 0xfa, 0x1a, 0x99, 0xf9, 0x3e, 0x54, 0x0e, 0x32, 0xf4,
- 0xd9, 0x32, 0xc5, 0x48, 0x3f, 0x0d, 0xa3, 0xa5, 0x91, 0x20, 0x43, 0xa4, 0xaa, 0x22, 0x92, 0x25,
- 0xa3, 0xa6, 0x24, 0x83, 0x85, 0x8c, 0xe7, 0xf4, 0x65, 0x1a, 0x32, 0x5b, 0x9b, 0x87, 0x30, 0x98,
- 0x50, 0x4c, 0xbd, 0x84, 0x7a, 0x4e, 0x92, 0xc2, 0x5c, 0x02, 0x54, 0x3b, 0x0f, 0x50, 0xfd, 0x34,
- 0x40, 0x2b, 0x19, 0xa0, 0xe6, 0x6f, 0x1a, 0x20, 0xf5, 0x26, 0x09, 0xc1, 0x3b, 0xb8, 0x8a, 0x41,
- 0x46, 0x43, 0x8a, 0x7d, 0x9b, 0xf7, 0x4a, 0xd9, 0xf1, 0xb8, 0x84, 0xb5, 0x63, 0x96, 0xa5, 0x79,
- 0x42, 0x5c, 0xa1, 0x15, 0xed, 0xae, 0xc1, 0x04, 0x5c, 0x59, 0xec, 0x96, 0xf5, 0x52, 0xb7, 0x34,
- 0xef, 0x42, 0x6b, 0x42, 0xc3, 0x18, 0x1f, 0x92, 0x67, 0x8b, 0xe8, 0x6d, 0xa2, 0x97, 0xd1, 0xe9,
- 0x39, 0x10, 0x23, 0x80, 0x9d, 0x3c, 0xfa, 0x55, 0x04, 0x78, 0x09, 0x2e, 0xe4, 0x16, 0x8c, 0x2f,
- 0x65, 0x5e, 0xcc, 0xa7, 0x70, 0xb1, 0xac, 0x90, 0x30, 0x7e, 0x04, 0xad, 0x1c, 0x92, 0x94, 0x3b,
- 0x2e, 0x28, 0x9f, 0x6c, 0x7e, 0xce, 0x52, 0x2d, 0xcd, 0xff, 0xc1, 0xa5, 0x5c, 0x75, 0x9f, 0x93,
- 0xe0, 0x59, 0xdc, 0x3c, 0x04, 0x63, 0xd9, 0x5c, 0xc4, 0x60, 0xfe, 0xa4, 0x43, 0xfb, 0xbe, 0xac,
- 0x76, 0xd6, 0x54, 0x95, 0x36, 0xda, 0xe4, 0x6d, 0xf4, 0x06, 0xb4, 0x0b, 0xa3, 0xab, 0x98, 0x62,
- 0x5a, 0xc7, 0xca, 0xdc, 0xba, 0x6a, 0xc2, 0xad, 0x70, 0xb3, 0xf2, 0x84, 0xfb, 0x6f, 0x18, 0x1c,
- 0xc4, 0x84, 0x2c, 0x0f, 0xc3, 0x55, 0xab, 0xc7, 0x14, 0xaa, 0xed, 0x16, 0x6c, 0x60, 0x87, 0x7a,
- 0xc7, 0x25, 0x6b, 0x91, 0xfb, 0x81, 0x50, 0xa9, 0xf6, 0x0f, 0xb2, 0x40, 0xbd, 0xe0, 0x20, 0x4c,
- 0x8c, 0xfa, 0xdb, 0x0f, 0xb3, 0xf2, 0x35, 0x4c, 0x93, 0x98, 0xdf, 0xeb, 0xd0, 0xb0, 0xb0, 0x73,
- 0xf4, 0x7e, 0xa3, 0x71, 0x07, 0x7a, 0x19, 0xab, 0x15, 0x00, 0xb9, 0xa4, 0x00, 0xa2, 0x26, 0xde,
- 0xea, 0xb8, 0xca, 0x2e, 0x31, 0xff, 0xd4, 0xa0, 0x7b, 0x3f, 0x63, 0xce, 0xf7, 0x1b, 0x8c, 0x6d,
- 0x00, 0x46, 0xf5, 0x05, 0x1c, 0xd4, 0xd6, 0x98, 0xa6, 0xdb, 0x6a, 0xc6, 0x72, 0x95, 0x98, 0x3f,
- 0xea, 0xd0, 0x7e, 0x16, 0x46, 0xa1, 0x1f, 0x1e, 0x2e, 0xde, 0xef, 0xd7, 0xef, 0xc2, 0x40, 0xe9,
- 0x8a, 0x05, 0x10, 0x2e, 0x97, 0x8a, 0x21, 0x4f, 0xb6, 0xd5, 0x73, 0x0b, 0xfb, 0xc4, 0xdc, 0x80,
- 0x81, 0x9c, 0xf0, 0x14, 0x72, 0xfb, 0x4e, 0x03, 0xa4, 0x4a, 0x25, 0xb3, 0x7d, 0x02, 0x1d, 0x2a,
- 0xb1, 0xe3, 0xf7, 0xc9, 0x21, 0x57, 0xad, 0x3d, 0x15, 0x5b, 0xab, 0x4d, 0x55, 0xa4, 0xff, 0x0f,
- 0x9b, 0xf2, 0x65, 0x8c, 0xed, 0x6d, 0x9f, 0xfd, 0x14, 0xb3, 0x67, 0x53, 0x89, 0xf0, 0xa0, 0xf4,
- 0x23, 0xed, 0xd1, 0x74, 0xfb, 0x87, 0x1a, 0xac, 0x4f, 0x08, 0x7e, 0x45, 0x88, 0x8b, 0xf6, 0xa0,
- 0x33, 0x21, 0x81, 0x9b, 0xff, 0x84, 0xdf, 0x54, 0x2e, 0xcd, 0xa4, 0xc3, 0x7f, 0xad, 0x92, 0x66,
- 0xac, 0xb8, 0x36, 0xd6, 0x6e, 0x6b, 0xe8, 0x09, 0x74, 0x1e, 0x12, 0x12, 0xed, 0x84, 0x41, 0x40,
- 0x1c, 0x4a, 0x5c, 0x74, 0x4d, 0xe5, 0xe6, 0xe5, 0xc1, 0x78, 0x78, 0x79, 0x89, 0x6c, 0xd2, 0x39,
- 0x4a, 0x7a, 0x7c, 0x0a, 0x6d, 0x75, 0x1e, 0x2c, 0x38, 0x5c, 0x31, 0xbd, 0x0e, 0xaf, 0x9f, 0x33,
- 0x48, 0x9a, 0x6b, 0xe8, 0x0e, 0xd4, 0xc5, 0x80, 0x82, 0x0c, 0xc5, 0xb8, 0x30, 0x81, 0x15, 0xe2,
- 0x2a, 0x4e, 0x33, 0xe6, 0x1a, 0x7a, 0x08, 0x90, 0xb7, 0x78, 0xa4, 0xe2, 0xb2, 0x34, 0x63, 0x0c,
- 0xaf, 0x9e, 0xa2, 0xcd, 0x9c, 0x7d, 0x01, 0xdd, 0x62, 0xb3, 0x43, 0xa3, 0x95, 0xfd, 0x4c, 0xa9,
- 0xa1, 0xe1, 0x8d, 0x33, 0x2c, 0x32, 0xc7, 0x5f, 0x41, 0xbf, 0xdc, 0xc3, 0x90, 0xb9, 0xf2, 0x60,
- 0xa1, 0x1f, 0x0e, 0x6f, 0x9e, 0x69, 0xa3, 0x82, 0x90, 0x97, 0x71, 0x01, 0x84, 0xa5, 0x9a, 0x2f,
- 0x80, 0xb0, 0x5c, 0xfb, 0xe6, 0xda, 0xb4, 0xce, 0xff, 0x54, 0xfa, 0xf0, 0xaf, 0x00, 0x00, 0x00,
- 0xff, 0xff, 0x94, 0xee, 0xe2, 0x89, 0x64, 0x12, 0x00, 0x00,
+ // 1572 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xd4, 0x58, 0xcd, 0x6e, 0xdc, 0x46,
+ 0x12, 0x16, 0xe7, 0x4f, 0x33, 0x35, 0x9a, 0xd1, 0x4c, 0x4b, 0xb6, 0xa9, 0xf1, 0xda, 0x1e, 0xd3,
+ 0x97, 0xb1, 0x77, 0x57, 0xeb, 0xd5, 0x1e, 0xf6, 0xb0, 0x1b, 0x18, 0xb6, 0x2c, 0x27, 0x82, 0xe5,
+ 0x3f, 0x8e, 0xed, 0x00, 0x01, 0x02, 0x86, 0x22, 0x4b, 0x32, 0x21, 0x0e, 0xc9, 0x90, 0x3d, 0xb2,
+ 0xc6, 0x97, 0x1c, 0x92, 0x63, 0x90, 0x1c, 0xf2, 0x12, 0x79, 0x88, 0x20, 0x97, 0x1c, 0xf3, 0x30,
+ 0x01, 0x72, 0x0f, 0x10, 0xf4, 0x0f, 0xc9, 0x26, 0x67, 0x24, 0xd9, 0x01, 0x7c, 0xf0, 0xad, 0xbb,
+ 0xaa, 0xba, 0xba, 0xfa, 0xeb, 0xea, 0xaf, 0x8a, 0x84, 0x95, 0x89, 0x9d, 0x50, 0x8c, 0x37, 0xa3,
+ 0x38, 0xa4, 0x21, 0x69, 0x89, 0x99, 0x15, 0xed, 0x1b, 0x3f, 0xd5, 0xa1, 0xf5, 0x09, 0xda, 0x31,
+ 0xdd, 0x47, 0x9b, 0x92, 0x2e, 0x54, 0xbc, 0x48, 0xd7, 0x86, 0xda, 0xa8, 0x65, 0x56, 0xbc, 0x88,
+ 0x10, 0xa8, 0x45, 0x61, 0x4c, 0xf5, 0xca, 0x50, 0x1b, 0x75, 0x4c, 0x3e, 0x26, 0x57, 0x00, 0xa2,
+ 0xe9, 0xbe, 0xef, 0x39, 0xd6, 0x34, 0xf6, 0xf5, 0x2a, 0xb7, 0x6d, 0x09, 0xc9, 0x8b, 0xd8, 0x27,
+ 0x23, 0xe8, 0x4d, 0xec, 0x13, 0xeb, 0x38, 0xf4, 0xa7, 0x13, 0xb4, 0x9c, 0x70, 0x1a, 0x50, 0xbd,
+ 0xc6, 0x97, 0x77, 0x27, 0xf6, 0xc9, 0x4b, 0x2e, 0xde, 0x66, 0x52, 0x32, 0x64, 0x51, 0x9d, 0x58,
+ 0x07, 0x9e, 0x8f, 0xd6, 0x11, 0xce, 0xf4, 0xfa, 0x50, 0x1b, 0xd5, 0x4c, 0x98, 0xd8, 0x27, 0x0f,
+ 0x3c, 0x1f, 0x1f, 0xe2, 0x8c, 0x5c, 0x83, 0xb6, 0x6b, 0x53, 0xdb, 0x72, 0x30, 0xa0, 0x18, 0xeb,
+ 0x0d, 0xbe, 0x17, 0x30, 0xd1, 0x36, 0x97, 0xb0, 0xf8, 0x62, 0xdb, 0x39, 0xd2, 0x97, 0xb9, 0x86,
+ 0x8f, 0x59, 0x7c, 0xb6, 0x3b, 0xf1, 0x02, 0x8b, 0x47, 0xde, 0xe4, 0x5b, 0xb7, 0xb8, 0xe4, 0x29,
+ 0x0b, 0xff, 0x23, 0x58, 0x16, 0xb1, 0x25, 0x7a, 0x6b, 0x58, 0x1d, 0xb5, 0xb7, 0x6e, 0x6c, 0x66,
+ 0x68, 0x6c, 0x8a, 0xf0, 0x76, 0x83, 0x83, 0x30, 0x9e, 0xd8, 0xd4, 0x0b, 0x83, 0x47, 0x98, 0x24,
+ 0xf6, 0x21, 0x9a, 0xe9, 0x1a, 0xb2, 0x0b, 0xed, 0x00, 0x5f, 0x5b, 0xa9, 0x0b, 0xe0, 0x2e, 0x46,
+ 0x73, 0x2e, 0xc6, 0xaf, 0xc2, 0x98, 0x2e, 0xf0, 0x03, 0x01, 0xbe, 0x7e, 0x29, 0x5d, 0x3d, 0x83,
+ 0x55, 0x17, 0x7d, 0xa4, 0xe8, 0x66, 0xee, 0xda, 0xef, 0xe8, 0xae, 0x2b, 0x1d, 0xa4, 0x2e, 0x3f,
+ 0x86, 0x16, 0x3a, 0x56, 0xf2, 0xca, 0x8e, 0xdd, 0x44, 0xef, 0x71, 0x67, 0xb7, 0xe6, 0x9c, 0xed,
+ 0x38, 0x63, 0x66, 0xb0, 0xc0, 0x5d, 0x13, 0x85, 0x2a, 0x21, 0x8f, 0xa1, 0xc3, 0x8e, 0x99, 0x3b,
+ 0xeb, 0xbf, 0xb3, 0x33, 0x86, 0xd3, 0x4e, 0xea, 0xef, 0x25, 0xf4, 0xd3, 0xb3, 0xe6, 0x3e, 0xc9,
+ 0x3b, 0xfb, 0x4c, 0x01, 0x4b, 0xfd, 0x1a, 0x2f, 0xa0, 0x9f, 0x65, 0xaf, 0x89, 0x49, 0x14, 0x06,
+ 0x09, 0x92, 0x11, 0xac, 0x0a, 0x40, 0xc7, 0xde, 0x1b, 0xdc, 0xf3, 0x26, 0x1e, 0xe5, 0x29, 0x5d,
+ 0x33, 0xcb, 0x62, 0x72, 0x11, 0x1a, 0x3e, 0xda, 0x2e, 0xc6, 0x32, 0x8f, 0xe5, 0xcc, 0xf8, 0xad,
+ 0x02, 0xfa, 0x69, 0xb9, 0xc0, 0x1f, 0x89, 0xcb, 0x3d, 0x76, 0xcc, 0x8a, 0xe7, 0xb2, 0x24, 0x4c,
+ 0xbc, 0x37, 0xc8, 0x1f, 0x49, 0xcd, 0xe4, 0x63, 0x72, 0x15, 0xc0, 0x09, 0x7d, 0x1f, 0x1d, 0xb6,
+ 0x50, 0x3a, 0x57, 0x24, 0x2c, 0x49, 0x79, 0xde, 0xe7, 0xef, 0xa3, 0x66, 0xb6, 0x98, 0x44, 0x3c,
+ 0x8d, 0xeb, 0xb0, 0x22, 0x4e, 0x2a, 0x0d, 0xc4, 0xd3, 0x68, 0x0b, 0x99, 0x30, 0xf9, 0x07, 0x90,
+ 0x14, 0xd1, 0xfd, 0x59, 0x66, 0xd8, 0xe0, 0x86, 0x3d, 0xa9, 0xb9, 0x37, 0x4b, 0xad, 0x2f, 0x43,
+ 0x2b, 0x46, 0xdb, 0xb5, 0xc2, 0xc0, 0x9f, 0xf1, 0xd7, 0xd2, 0x34, 0x9b, 0x4c, 0xf0, 0x24, 0xf0,
+ 0x67, 0xe4, 0xef, 0xd0, 0x8f, 0x31, 0xf2, 0x3d, 0xc7, 0xb6, 0x22, 0xdf, 0x76, 0x70, 0x82, 0x41,
+ 0xfa, 0x70, 0x7a, 0x52, 0xf1, 0x34, 0x95, 0x13, 0x1d, 0x96, 0x8f, 0x31, 0x4e, 0xd8, 0xb1, 0x5a,
+ 0xdc, 0x24, 0x9d, 0x92, 0x1e, 0x54, 0x29, 0xf5, 0x75, 0xe0, 0x52, 0x36, 0x24, 0x37, 0xa1, 0xe7,
+ 0x84, 0x93, 0xc8, 0x76, 0xa8, 0x15, 0xe3, 0xb1, 0xc7, 0x17, 0xb5, 0xb9, 0x7a, 0x55, 0xca, 0x4d,
+ 0x29, 0x36, 0x7e, 0xd4, 0xe0, 0xca, 0x99, 0xb9, 0x3e, 0x07, 0xfb, 0x79, 0x10, 0xbf, 0xaf, 0x53,
+ 0x19, 0x3e, 0x5c, 0x3b, 0x27, 0x4f, 0xcf, 0x89, 0xb5, 0x32, 0x17, 0xeb, 0x06, 0x34, 0xd1, 0xb1,
+ 0xbc, 0xc0, 0xc5, 0x13, 0x7e, 0x92, 0x8e, 0xb9, 0x8c, 0xce, 0x2e, 0x9b, 0x1a, 0xcb, 0x50, 0xdf,
+ 0x99, 0x44, 0x74, 0x66, 0xfc, 0xac, 0xc1, 0xea, 0x78, 0x1a, 0x61, 0x7c, 0xcf, 0x0f, 0x9d, 0xa3,
+ 0x9d, 0x13, 0x1a, 0xdb, 0xe4, 0x09, 0x74, 0x31, 0xb6, 0x93, 0x69, 0xcc, 0xee, 0xdf, 0xf5, 0x82,
+ 0x43, 0xbe, 0x67, 0x91, 0x41, 0x4a, 0x6b, 0x36, 0x77, 0xc4, 0x82, 0x6d, 0x6e, 0x6f, 0x76, 0x50,
+ 0x9d, 0x0e, 0x3e, 0x83, 0x4e, 0x41, 0xcf, 0x92, 0x9b, 0xf1, 0xad, 0x3c, 0x0b, 0x1f, 0xb3, 0x57,
+ 0x13, 0xd9, 0xb1, 0x47, 0x67, 0xb2, 0x2e, 0xc8, 0x19, 0x4b, 0x6a, 0x49, 0xfb, 0x9e, 0x9b, 0xe8,
+ 0xd5, 0x61, 0x95, 0x31, 0xaf, 0x90, 0xec, 0xba, 0x89, 0x71, 0x13, 0xd6, 0xb6, 0x7d, 0x0f, 0x03,
+ 0xba, 0xe7, 0x25, 0x14, 0x03, 0x13, 0xbf, 0x9c, 0x62, 0x42, 0xd9, 0x0e, 0x81, 0x3d, 0x41, 0x59,
+ 0x75, 0xf8, 0xd8, 0xf8, 0x0a, 0xba, 0x02, 0xe2, 0xbd, 0xd0, 0xe1, 0xc0, 0xb2, 0x6b, 0x60, 0xe5,
+ 0x46, 0x18, 0xb1, 0x61, 0xa9, 0x0e, 0x55, 0xca, 0x75, 0x68, 0x03, 0x9a, 0x9c, 0xa8, 0xf3, 0x50,
+ 0x96, 0x19, 0xf7, 0x7a, 0x6e, 0x92, 0xbf, 0x2e, 0x57, 0xa8, 0x6b, 0x5c, 0xdd, 0x4e, 0xb9, 0xd4,
+ 0x73, 0x13, 0xe3, 0x39, 0xac, 0xed, 0x85, 0xe1, 0xd1, 0x34, 0x12, 0x61, 0xa4, 0xb1, 0x16, 0x4f,
+ 0xa8, 0x0d, 0xab, 0x6c, 0xcf, 0xec, 0x84, 0xe7, 0x5d, 0xb3, 0xf1, 0xbb, 0x06, 0xeb, 0x45, 0xb7,
+ 0x92, 0xb1, 0xbe, 0x80, 0xb5, 0xcc, 0xaf, 0xe5, 0xcb, 0x33, 0x8b, 0x0d, 0xda, 0x5b, 0xb7, 0x95,
+ 0xcb, 0x5c, 0xb4, 0x3a, 0xad, 0x5a, 0x6e, 0x0a, 0x96, 0xd9, 0x3f, 0x2e, 0x49, 0x92, 0xc1, 0x09,
+ 0xf4, 0xca, 0x66, 0x8c, 0x14, 0xb2, 0x5d, 0x25, 0xb2, 0xcd, 0x74, 0x25, 0xf9, 0x37, 0xb4, 0xf2,
+ 0x40, 0x2a, 0x3c, 0x90, 0xb5, 0x42, 0x20, 0x72, 0xaf, 0xdc, 0x8a, 0xac, 0x43, 0x1d, 0xe3, 0x38,
+ 0x4c, 0xc9, 0x54, 0x4c, 0x8c, 0xff, 0x41, 0xf3, 0x2f, 0xdf, 0xa2, 0xf1, 0xab, 0x06, 0x9d, 0xbb,
+ 0x49, 0xe2, 0x1d, 0x66, 0xe9, 0xb2, 0x0e, 0x75, 0x41, 0x75, 0x82, 0xd2, 0xc5, 0x84, 0x0c, 0xa1,
+ 0x2d, 0xdf, 0xb4, 0x02, 0xbd, 0x2a, 0x3a, 0x97, 0x2e, 0xe4, 0x3b, 0xaf, 0x89, 0xd0, 0x18, 0x7b,
+ 0x95, 0xba, 0x8f, 0xfa, 0xa9, 0xdd, 0x47, 0x43, 0xe9, 0x3e, 0x2e, 0x43, 0x8b, 0x2f, 0x0a, 0x42,
+ 0x17, 0x65, 0x5b, 0xd2, 0x64, 0x82, 0xc7, 0xa1, 0x8b, 0xc6, 0x0f, 0x1a, 0x74, 0xd3, 0xd3, 0xc8,
+ 0x9b, 0xef, 0x41, 0xf5, 0x20, 0x43, 0x9f, 0x0d, 0x53, 0x8c, 0x2a, 0xa7, 0x61, 0x34, 0xd7, 0x71,
+ 0x65, 0x88, 0xd4, 0x54, 0x44, 0xb2, 0xcb, 0xa8, 0x2b, 0x97, 0xc1, 0x42, 0xb6, 0xa7, 0xf4, 0x55,
+ 0x1a, 0x32, 0x1b, 0x1b, 0x87, 0xd0, 0x1f, 0x53, 0x9b, 0x7a, 0x09, 0xf5, 0x9c, 0x24, 0x85, 0xb9,
+ 0x04, 0xa8, 0x76, 0x1e, 0xa0, 0x95, 0xd3, 0x00, 0xad, 0x66, 0x80, 0x1a, 0xbf, 0x68, 0x40, 0xd4,
+ 0x9d, 0x24, 0x04, 0xef, 0x61, 0x2b, 0x06, 0x19, 0x0d, 0xa9, 0xed, 0x5b, 0xbc, 0x32, 0xcb, 0xfa,
+ 0xca, 0x25, 0xac, 0xf8, 0xb3, 0x5b, 0x9a, 0x26, 0xe8, 0x0a, 0xad, 0x28, 0xae, 0x4d, 0x26, 0xe0,
+ 0xca, 0x62, 0x6d, 0x6e, 0x94, 0x6a, 0xb3, 0x71, 0x17, 0xda, 0x63, 0x1a, 0xc6, 0xf6, 0x21, 0x3e,
+ 0x9f, 0x45, 0x6f, 0x13, 0xbd, 0x8c, 0xae, 0x92, 0x03, 0x31, 0x04, 0xd8, 0xce, 0xa3, 0x5f, 0x44,
+ 0x80, 0x97, 0xe0, 0x42, 0x6e, 0xc1, 0xf8, 0x52, 0xde, 0x8b, 0xf1, 0x0c, 0x2e, 0x96, 0x15, 0x12,
+ 0xc6, 0xff, 0x42, 0x3b, 0x87, 0x24, 0xe5, 0x8e, 0x0b, 0xca, 0x93, 0xcd, 0xd7, 0x99, 0xaa, 0xa5,
+ 0xf1, 0x4f, 0xb8, 0x94, 0xab, 0xee, 0x73, 0x12, 0x3c, 0x8b, 0x9b, 0x07, 0xa0, 0xcf, 0x9b, 0x8b,
+ 0x18, 0x8c, 0xef, 0x2b, 0xb0, 0x72, 0x5f, 0x66, 0x3b, 0x2b, 0x8b, 0x4a, 0x21, 0x6c, 0xf1, 0x42,
+ 0x78, 0x1d, 0x56, 0x0a, 0x5f, 0x06, 0xa2, 0x67, 0x6a, 0x1f, 0x2b, 0x9f, 0x05, 0x8b, 0x3e, 0x20,
+ 0xaa, 0xdc, 0xac, 0xfc, 0x01, 0x71, 0x0b, 0xfa, 0x07, 0x31, 0xe2, 0xfc, 0xb7, 0x46, 0xcd, 0x5c,
+ 0x65, 0x0a, 0xd5, 0x76, 0x13, 0xd6, 0x6c, 0x87, 0x7a, 0xc7, 0x25, 0x6b, 0x71, 0xf7, 0x7d, 0xa1,
+ 0x52, 0xed, 0x1f, 0x64, 0x81, 0x7a, 0xc1, 0x41, 0x98, 0xe8, 0x8d, 0xb7, 0xff, 0x56, 0x90, 0xa7,
+ 0x61, 0x9a, 0xc4, 0xf8, 0xa6, 0x02, 0x4d, 0xd3, 0x76, 0x8e, 0x3e, 0x6c, 0x34, 0xee, 0xc0, 0x6a,
+ 0xc6, 0x6a, 0x05, 0x40, 0x2e, 0x29, 0x80, 0xa8, 0x17, 0x6f, 0x76, 0x5c, 0x65, 0x96, 0x18, 0x7f,
+ 0x68, 0xd0, 0xbd, 0x9f, 0x31, 0xe7, 0x87, 0x0d, 0xc6, 0x16, 0x00, 0xa3, 0xfa, 0x02, 0x0e, 0x6a,
+ 0x69, 0x4c, 0xaf, 0xdb, 0x6c, 0xc5, 0x72, 0x94, 0x18, 0xdf, 0x55, 0x60, 0xe5, 0x79, 0x18, 0x85,
+ 0x7e, 0x78, 0x38, 0xfb, 0xb0, 0x4f, 0xbf, 0x03, 0x7d, 0xa5, 0x2a, 0x16, 0x40, 0xd8, 0x28, 0x25,
+ 0x43, 0x7e, 0xd9, 0xe6, 0xaa, 0x5b, 0x98, 0x27, 0xc6, 0x1a, 0xf4, 0x65, 0x87, 0xa7, 0x90, 0xdb,
+ 0xd7, 0x1a, 0x10, 0x55, 0x2a, 0x99, 0xed, 0xff, 0xd0, 0xa1, 0x12, 0x3b, 0xbe, 0x9f, 0x6c, 0x72,
+ 0xd5, 0xdc, 0x53, 0xb1, 0x35, 0x57, 0xa8, 0x8a, 0xf4, 0xbf, 0x60, 0x5d, 0x9e, 0x8c, 0xb1, 0xbd,
+ 0xe5, 0xb3, 0x0f, 0x3f, 0x6b, 0xb2, 0x2f, 0x11, 0xee, 0x97, 0x3e, 0x09, 0x1f, 0xed, 0x6f, 0x7d,
+ 0x5b, 0x87, 0xe5, 0x31, 0xda, 0xaf, 0x11, 0x5d, 0xb2, 0x0b, 0x9d, 0x31, 0x06, 0x6e, 0xfe, 0x87,
+ 0x64, 0x5d, 0xd9, 0x34, 0x93, 0x0e, 0xfe, 0xb6, 0x48, 0x9a, 0xb1, 0xe2, 0xd2, 0x48, 0xbb, 0xad,
+ 0x91, 0xa7, 0xd0, 0x79, 0x88, 0x18, 0x6d, 0x87, 0x41, 0x80, 0x0e, 0x45, 0x97, 0x5c, 0x55, 0xb9,
+ 0x79, 0xbe, 0x31, 0x1e, 0x6c, 0xcc, 0x91, 0x4d, 0xda, 0x47, 0x49, 0x8f, 0xcf, 0x60, 0x45, 0xed,
+ 0x07, 0x0b, 0x0e, 0x17, 0x74, 0xaf, 0x83, 0x6b, 0xe7, 0x34, 0x92, 0xc6, 0x12, 0xb9, 0x03, 0x0d,
+ 0xd1, 0xa0, 0x10, 0x5d, 0x31, 0x2e, 0x74, 0x60, 0x85, 0xb8, 0x8a, 0xdd, 0x8c, 0xb1, 0x44, 0x1e,
+ 0x02, 0xe4, 0x25, 0x9e, 0xa8, 0xb8, 0xcc, 0xf5, 0x18, 0x83, 0x2b, 0xa7, 0x68, 0x33, 0x67, 0x9f,
+ 0x42, 0xb7, 0x58, 0xec, 0xc8, 0x70, 0x61, 0x3d, 0x53, 0x72, 0x68, 0x70, 0xfd, 0x0c, 0x8b, 0xcc,
+ 0xf1, 0xe7, 0xd0, 0x2b, 0xd7, 0x30, 0x62, 0x2c, 0x5c, 0x58, 0xa8, 0x87, 0x83, 0x1b, 0x67, 0xda,
+ 0xa8, 0x20, 0xe4, 0x69, 0x5c, 0x00, 0x61, 0x2e, 0xe7, 0x0b, 0x20, 0xcc, 0xe7, 0xbe, 0xb1, 0xb4,
+ 0xdf, 0xe0, 0xff, 0xec, 0xfe, 0xf3, 0x67, 0x00, 0x00, 0x00, 0xff, 0xff, 0x63, 0xc4, 0xec, 0xbe,
+ 0xc3, 0x13, 0x00, 0x00,
}
diff --git a/weed/server/master_grpc_server.go b/weed/server/master_grpc_server.go
index 1cc3fef7a..34962c83c 100644
--- a/weed/server/master_grpc_server.go
+++ b/weed/server/master_grpc_server.go
@@ -87,7 +87,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
}
// update master internal volume layouts
t.IncrementalSyncDataNodeRegistration(heartbeat.NewVolumes, heartbeat.DeletedVolumes, dn)
- } else {
+ } else if len(heartbeat.Volumes) > 0 {
// process heartbeat.Volumes
newVolumes, deletedVolumes := t.SyncDataNodeRegistration(heartbeat.Volumes, dn)
@@ -99,6 +99,8 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
glog.V(0).Infof("master see deleted volume %d from %s", uint32(v.Id), dn.Url())
message.DeletedVids = append(message.DeletedVids, uint32(v.Id))
}
+ } else if len(heartbeat.EcShards) > 0 {
+ glog.V(0).Infof("master recieved ec shards from %s: %+v", dn.Url(), heartbeat.EcShards)
}
if len(message.NewVids) > 0 || len(message.DeletedVids) > 0 {
diff --git a/weed/server/volume_grpc_client_to_master.go b/weed/server/volume_grpc_client_to_master.go
index 0d4532485..6364cbc10 100644
--- a/weed/server/volume_grpc_client_to_master.go
+++ b/weed/server/volume_grpc_client_to_master.go
@@ -94,7 +94,13 @@ func (vs *VolumeServer) doHeartbeat(ctx context.Context, masterNode, masterGrpcA
return "", err
}
- tickChan := time.Tick(sleepInterval)
+ if err = stream.Send(vs.store.CollectErasureCodingHeartbeat()); err != nil {
+ glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err)
+ return "", err
+ }
+
+ volumeTickChan := time.Tick(sleepInterval)
+ ecShardTickChan := time.Tick(17 * sleepInterval)
for {
select {
@@ -109,6 +115,17 @@ func (vs *VolumeServer) doHeartbeat(ctx context.Context, masterNode, masterGrpcA
glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err)
return "", err
}
+ case ecShardMessage := <-vs.store.NewEcShardsChan:
+ deltaBeat := &master_pb.Heartbeat{
+ NewEcShards: []*master_pb.VolumeEcShardInformationMessage{
+ &ecShardMessage,
+ },
+ }
+ glog.V(1).Infof("volume server %s:%d adds ec shard %d:%d", vs.store.Ip, vs.store.Port, ecShardMessage.Id, ecShardMessage.EcIndex)
+ if err = stream.Send(deltaBeat); err != nil {
+ glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err)
+ return "", err
+ }
case volumeMessage := <-vs.store.DeletedVolumesChan:
deltaBeat := &master_pb.Heartbeat{
DeletedVolumes: []*master_pb.VolumeShortInformationMessage{
@@ -120,12 +137,29 @@ func (vs *VolumeServer) doHeartbeat(ctx context.Context, masterNode, masterGrpcA
glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err)
return "", err
}
- case <-tickChan:
+ case ecShardMessage := <-vs.store.DeletedEcShardsChan:
+ deltaBeat := &master_pb.Heartbeat{
+ DeletedEcShards: []*master_pb.VolumeEcShardInformationMessage{
+ &ecShardMessage,
+ },
+ }
+ glog.V(1).Infof("volume server %s:%d deletes ec shard %d:%d", vs.store.Ip, vs.store.Port, ecShardMessage.Id, ecShardMessage.EcIndex)
+ if err = stream.Send(deltaBeat); err != nil {
+ glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err)
+ return "", err
+ }
+ case <-volumeTickChan:
glog.V(4).Infof("volume server %s:%d heartbeat", vs.store.Ip, vs.store.Port)
if err = stream.Send(vs.store.CollectHeartbeat()); err != nil {
glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err)
return "", err
}
+ case <-ecShardTickChan:
+ glog.V(4).Infof("volume server %s:%d ec heartbeat", vs.store.Ip, vs.store.Port)
+ if err = stream.Send(vs.store.CollectErasureCodingHeartbeat()); err != nil {
+ glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err)
+ return "", err
+ }
case err = <-doneChan:
return
}
diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go
index a4a3c519e..bd6d467d9 100644
--- a/weed/storage/disk_location.go
+++ b/weed/storage/disk_location.go
@@ -9,6 +9,7 @@ import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
+ "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
)
@@ -17,43 +18,52 @@ type DiskLocation struct {
MaxVolumeCount int
volumes map[needle.VolumeId]*Volume
sync.RWMutex
+
+ // erasure coding
+ ecShards map[needle.VolumeId]erasure_coding.EcVolumeShards
+ ecShardsLock sync.RWMutex
}
func NewDiskLocation(dir string, maxVolumeCount int) *DiskLocation {
location := &DiskLocation{Directory: dir, MaxVolumeCount: maxVolumeCount}
location.volumes = make(map[needle.VolumeId]*Volume)
+ location.ecShards = make(map[needle.VolumeId]erasure_coding.EcVolumeShards)
return location
}
func (l *DiskLocation) volumeIdFromPath(dir os.FileInfo) (needle.VolumeId, string, error) {
name := dir.Name()
if !dir.IsDir() && strings.HasSuffix(name, ".dat") {
- collection := ""
base := name[:len(name)-len(".dat")]
- i := strings.LastIndex(base, "_")
- if i > 0 {
- collection, base = base[0:i], base[i+1:]
- }
- vol, err := needle.NewVolumeId(base)
- return vol, collection, err
+ collection, volumeId, err := parseCollectionVolumeId(base)
+ return volumeId, collection, err
}
return 0, "", fmt.Errorf("Path is not a volume: %s", name)
}
-func (l *DiskLocation) loadExistingVolume(dir os.FileInfo, needleMapKind NeedleMapType, mutex *sync.RWMutex) {
- name := dir.Name()
- if !dir.IsDir() && strings.HasSuffix(name, ".dat") {
- vid, collection, err := l.volumeIdFromPath(dir)
+func parseCollectionVolumeId(base string) (collection string, vid needle.VolumeId, err error) {
+ i := strings.LastIndex(base, "_")
+ if i > 0 {
+ collection, base = base[0:i], base[i+1:]
+ }
+ vol, err := needle.NewVolumeId(base)
+ return collection, vol, err
+}
+
+func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind NeedleMapType) {
+ name := fileInfo.Name()
+ if !fileInfo.IsDir() && strings.HasSuffix(name, ".dat") {
+ vid, collection, err := l.volumeIdFromPath(fileInfo)
if err == nil {
- mutex.RLock()
+ l.RLock()
_, found := l.volumes[vid]
- mutex.RUnlock()
+ l.RUnlock()
if !found {
if v, e := NewVolume(l.Directory, collection, vid, needleMapKind, nil, nil, 0); e == nil {
- mutex.Lock()
+ l.Lock()
l.volumes[vid] = v
- mutex.Unlock()
+ l.Unlock()
size, _, _ := v.FileStat()
glog.V(0).Infof("data file %s, replicaPlacement=%s v=%d size=%d ttl=%s",
l.Directory+"/"+name, v.ReplicaPlacement, v.Version(), size, v.Ttl.String())
@@ -80,13 +90,12 @@ func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapType, con
}()
var wg sync.WaitGroup
- var mutex sync.RWMutex
for workerNum := 0; workerNum < concurrency; workerNum++ {
wg.Add(1)
go func() {
defer wg.Done()
for dir := range task_queue {
- l.loadExistingVolume(dir, needleMapKind, &mutex)
+ l.loadExistingVolume(dir, needleMapKind)
}
}()
}
@@ -95,12 +104,13 @@ func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapType, con
}
func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapType) {
- l.Lock()
- defer l.Unlock()
l.concurrentLoadingVolumes(needleMapKind, 10)
+ glog.V(0).Infof("Store started on dir: %s with %d volumes max %d", l.Directory, len(l.volumes), l.MaxVolumeCount)
+
+ l.loadAllEcShards()
+ glog.V(0).Infof("Store started on dir: %s with %d ec shards", l.Directory, len(l.ecShards))
- glog.V(0).Infoln("Store started on dir:", l.Directory, "with", len(l.volumes), "volumes", "max", l.MaxVolumeCount)
}
func (l *DiskLocation) DeleteCollectionFromDiskLocation(collection string) (e error) {
@@ -132,12 +142,11 @@ func (l *DiskLocation) deleteVolumeById(vid needle.VolumeId) (e error) {
}
func (l *DiskLocation) LoadVolume(vid needle.VolumeId, needleMapKind NeedleMapType) bool {
- if dirs, err := ioutil.ReadDir(l.Directory); err == nil {
- for _, dir := range dirs {
- volId, _, err := l.volumeIdFromPath(dir)
+ if fileInfos, err := ioutil.ReadDir(l.Directory); err == nil {
+ for _, fileInfo := range fileInfos {
+ volId, _, err := l.volumeIdFromPath(fileInfo)
if vid == volId && err == nil {
- var mutex sync.RWMutex
- l.loadExistingVolume(dir, needleMapKind, &mutex)
+ l.loadExistingVolume(fileInfo, needleMapKind)
return true
}
}
@@ -194,10 +203,16 @@ func (l *DiskLocation) VolumesLen() int {
func (l *DiskLocation) Close() {
l.Lock()
- defer l.Unlock()
-
for _, v := range l.volumes {
v.Close()
}
+ l.Unlock()
+
+ l.ecShardsLock.Lock()
+ for _, shards := range l.ecShards {
+ shards.Close()
+ }
+ l.ecShardsLock.Unlock()
+
return
}
diff --git a/weed/storage/disk_location_ec.go b/weed/storage/disk_location_ec.go
new file mode 100644
index 000000000..3b2f1ec02
--- /dev/null
+++ b/weed/storage/disk_location_ec.go
@@ -0,0 +1,84 @@
+package storage
+
+import (
+ "fmt"
+ "io/ioutil"
+ "path"
+ "regexp"
+ "sort"
+ "strconv"
+
+ "github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
+ "github.com/chrislusf/seaweedfs/weed/storage/needle"
+)
+
+var (
+ re = regexp.MustCompile("\\.ec[0-9][0-9]")
+)
+
+func (l *DiskLocation) loadEcShards(baseName string, shards []string, collection string, vid needle.VolumeId) (err error){
+
+ for _, shard := range shards{
+ shardId, err := strconv.ParseInt(path.Ext(shard)[3:], 10, 64)
+ if err != nil {
+ return fmt.Errorf("failed to parse ec shard name %v: %v", shard, err)
+ }
+ ecVolumeShard, err := erasure_coding.NewEcVolumeShard(l.Directory, collection, vid, int(shardId))
+ if err != nil {
+ return fmt.Errorf("failed to create ec shard %v: %v", shard, err)
+ }
+ l.ecShardsLock.Lock()
+ l.ecShards[vid] = append(l.ecShards[vid], ecVolumeShard)
+ l.ecShardsLock.Unlock()
+ }
+
+ return nil
+}
+
+func (l *DiskLocation) loadAllEcShards() (err error){
+
+ fileInfos, err := ioutil.ReadDir(l.Directory)
+ if err != nil {
+ return fmt.Errorf("load all ec shards in dir %s: %v", l.Directory, err)
+ }
+
+ sort.Slice(fileInfos, func(i, j int) bool {
+ return fileInfos[i].Name() < fileInfos[j].Name()
+ })
+
+ var sameVolumeShards []string
+ var prevVolumeId needle.VolumeId
+ for _, fileInfo := range fileInfos{
+ if fileInfo.IsDir(){
+ continue
+ }
+ ext := path.Ext(fileInfo.Name())
+ name := fileInfo.Name()
+ baseName := name[:len(name)-len(ext)]
+
+ collection, volumeId, err := parseCollectionVolumeId(baseName)
+ if err != nil {
+ continue
+ }
+
+ if re.MatchString(ext){
+ if prevVolumeId == 0 || volumeId == prevVolumeId {
+ sameVolumeShards = append(sameVolumeShards, fileInfo.Name())
+ }else{
+ sameVolumeShards = []string{fileInfo.Name()}
+ }
+ prevVolumeId = volumeId
+ continue
+ }
+
+ if ext == ".ecx" && volumeId == prevVolumeId{
+ if err = l.loadEcShards(baseName, sameVolumeShards, collection, volumeId);err!=nil{
+ return fmt.Errorf("loadEcShards collection:%v volumeId:%d : %v", collection, volumeId, err)
+ }
+ prevVolumeId = volumeId
+ continue
+ }
+
+ }
+ return nil
+}
diff --git a/weed/storage/disk_location_ec_test.go b/weed/storage/disk_location_ec_test.go
new file mode 100644
index 000000000..99cb242a4
--- /dev/null
+++ b/weed/storage/disk_location_ec_test.go
@@ -0,0 +1,17 @@
+package storage
+
+import (
+ "testing"
+)
+
+func TestLoadingEcShards(t *testing.T) {
+ dl := NewDiskLocation("./erasure_coding", 100)
+ err := dl.loadAllEcShards()
+ if err != nil {
+ t.Errorf("load all ec shards: %v", err)
+ }
+
+ if len(dl.ecShards)!=1 {
+ t.Errorf("loading err")
+ }
+} \ No newline at end of file
diff --git a/weed/storage/erasure_coding/3.ec07 b/weed/storage/erasure_coding/3.ec07
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/weed/storage/erasure_coding/3.ec07
diff --git a/weed/storage/erasure_coding/3.ecx b/weed/storage/erasure_coding/3.ecx
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/weed/storage/erasure_coding/3.ecx
diff --git a/weed/storage/erasure_coding/ec_encoder.go b/weed/storage/erasure_coding/ec_encoder.go
index f4aa17e9d..54e757678 100644
--- a/weed/storage/erasure_coding/ec_encoder.go
+++ b/weed/storage/erasure_coding/ec_encoder.go
@@ -6,7 +6,7 @@ import (
"os"
"github.com/chrislusf/seaweedfs/weed/glog"
- "github.com/chrislusf/seaweedfs/weed/storage"
+ "github.com/chrislusf/seaweedfs/weed/storage/idx"
"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
"github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/klauspost/reedsolomon"
@@ -190,7 +190,7 @@ func readCompactMap(baseFileName string) (*needle_map.CompactMap, error) {
defer indexFile.Close()
cm := needle_map.NewCompactMap()
- err = storage.WalkIndexFile(indexFile, func(key types.NeedleId, offset types.Offset, size uint32) error {
+ err = idx.WalkIndexFile(indexFile, func(key types.NeedleId, offset types.Offset, size uint32) error {
if !offset.IsZero() && size != types.TombstoneFileSize {
cm.Set(key, offset, size)
} else {
diff --git a/weed/storage/erasure_coding/ec_volume.go b/weed/storage/erasure_coding/ec_volume.go
new file mode 100644
index 000000000..db0f911e4
--- /dev/null
+++ b/weed/storage/erasure_coding/ec_volume.go
@@ -0,0 +1,108 @@
+package erasure_coding
+
+import (
+ "fmt"
+ "os"
+ "path"
+ "strconv"
+
+ "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+ "github.com/chrislusf/seaweedfs/weed/storage/needle"
+)
+
+type EcVolumeShard struct {
+ VolumeId needle.VolumeId
+ ShardId uint8
+ Collection string
+ dir string
+ ecdFile *os.File
+ ecxFile *os.File
+}
+type EcVolumeShards []*EcVolumeShard
+
+func NewEcVolumeShard(dirname string, collection string, id needle.VolumeId, shardId int) (v *EcVolumeShard, e error) {
+
+ v = &EcVolumeShard{dir: dirname, Collection: collection, VolumeId: id, ShardId: uint8(shardId)}
+
+ baseFileName := v.FileName()
+ if v.ecxFile, e = os.OpenFile(baseFileName+".ecx", os.O_RDONLY, 0644); e != nil {
+ return nil, fmt.Errorf("cannot read ec volume index %s.ecx: %v", baseFileName, e)
+ }
+ if v.ecdFile, e = os.OpenFile(baseFileName+ToExt(shardId), os.O_RDONLY, 0644); e != nil {
+ return nil, fmt.Errorf("cannot read ec volume shard %s.%s: %v", baseFileName, ToExt(shardId), e)
+ }
+
+ return
+}
+
+func (shards *EcVolumeShards) AddEcVolumeShard(ecVolumeShard *EcVolumeShard) bool {
+ for _, s := range *shards {
+ if s.ShardId == ecVolumeShard.ShardId {
+ return false
+ }
+ }
+ *shards = append(*shards, ecVolumeShard)
+ return true
+}
+
+func (shards *EcVolumeShards) DeleteEcVolumeShard(ecVolumeShard *EcVolumeShard) bool {
+ foundPosition := -1
+ for i, s := range *shards {
+ if s.ShardId == ecVolumeShard.ShardId {
+ foundPosition = i
+ }
+ }
+ if foundPosition < 0 {
+ return false
+ }
+
+ *shards = append((*shards)[:foundPosition], (*shards)[foundPosition+1:]...)
+ return true
+}
+
+func (shards *EcVolumeShards) Close() {
+ for _, s := range *shards {
+ s.Close()
+ }
+}
+
+func (shards *EcVolumeShards) ToVolumeInformationMessage() (messages []*master_pb.VolumeEcShardInformationMessage) {
+ for _, s := range *shards {
+ m := &master_pb.VolumeEcShardInformationMessage{
+ Id: uint32(s.VolumeId),
+ Collection: s.Collection,
+ EcIndex: uint32(s.ShardId),
+ }
+ messages = append(messages, m)
+ }
+ return
+}
+
+func (v *EcVolumeShard) String() string {
+ return fmt.Sprintf("ec shard %v:%v, dir:%s, Collection:%s", v.VolumeId, v.ShardId, v.dir, v.Collection)
+}
+
+func (v *EcVolumeShard) FileName() (fileName string) {
+ return EcShardFileName(v.Collection, v.dir, int(v.VolumeId))
+}
+
+func EcShardFileName(collection string, dir string, id int) (fileName string) {
+ idString := strconv.Itoa(id)
+ if collection == "" {
+ fileName = path.Join(dir, idString)
+ } else {
+ fileName = path.Join(dir, collection+"_"+idString)
+ }
+ return
+}
+
+func (v *EcVolumeShard) Close() {
+ if v.ecdFile != nil {
+ _ = v.ecdFile.Close()
+ v.ecdFile = nil
+ }
+ if v.ecxFile != nil {
+ _ = v.ecxFile.Close()
+ v.ecxFile = nil
+ }
+}
diff --git a/weed/storage/idx/walk.go b/weed/storage/idx/walk.go
new file mode 100644
index 000000000..c674a2aeb
--- /dev/null
+++ b/weed/storage/idx/walk.go
@@ -0,0 +1,54 @@
+package idx
+
+import (
+ "io"
+ "os"
+
+ "github.com/chrislusf/seaweedfs/weed/glog"
+ "github.com/chrislusf/seaweedfs/weed/storage/types"
+ "github.com/chrislusf/seaweedfs/weed/util"
+)
+
+// walks through the index file, calls fn function with each key, offset, size
+// stops with the error returned by the fn function
+func WalkIndexFile(r *os.File, fn func(key types.NeedleId, offset types.Offset, size uint32) error) error {
+ var readerOffset int64
+ bytes := make([]byte, types.NeedleMapEntrySize*RowsToRead)
+ count, e := r.ReadAt(bytes, readerOffset)
+ glog.V(3).Infoln("file", r.Name(), "readerOffset", readerOffset, "count", count, "e", e)
+ readerOffset += int64(count)
+ var (
+ key types.NeedleId
+ offset types.Offset
+ size uint32
+ i int
+ )
+
+ for count > 0 && e == nil || e == io.EOF {
+ for i = 0; i+types.NeedleMapEntrySize <= count; i += types.NeedleMapEntrySize {
+ key, offset, size = IdxFileEntry(bytes[i : i+types.NeedleMapEntrySize])
+ if e = fn(key, offset, size); e != nil {
+ return e
+ }
+ }
+ if e == io.EOF {
+ return nil
+ }
+ count, e = r.ReadAt(bytes, readerOffset)
+ glog.V(3).Infoln("file", r.Name(), "readerOffset", readerOffset, "count", count, "e", e)
+ readerOffset += int64(count)
+ }
+ return e
+}
+
+func IdxFileEntry(bytes []byte) (key types.NeedleId, offset types.Offset, size uint32) {
+ key = types.BytesToNeedleId(bytes[:types.NeedleIdSize])
+ offset = types.BytesToOffset(bytes[types.NeedleIdSize : types.NeedleIdSize+types.OffsetSize])
+ size = util.BytesToUint32(bytes[types.NeedleIdSize+types.OffsetSize : types.NeedleIdSize+types.OffsetSize+types.SizeSize])
+ return
+}
+
+const (
+ RowsToRead = 1024
+)
+
diff --git a/weed/storage/needle_map.go b/weed/storage/needle_map.go
index f2c88093f..876839be1 100644
--- a/weed/storage/needle_map.go
+++ b/weed/storage/needle_map.go
@@ -8,7 +8,6 @@ import (
"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
- "github.com/chrislusf/seaweedfs/weed/util"
)
type NeedleMapType int
@@ -55,12 +54,6 @@ func (nm *baseNeedleMapper) IndexFileName() string {
return nm.indexFile.Name()
}
-func IdxFileEntry(bytes []byte) (key NeedleId, offset Offset, size uint32) {
- key = BytesToNeedleId(bytes[:NeedleIdSize])
- offset = BytesToOffset(bytes[NeedleIdSize : NeedleIdSize+OffsetSize])
- size = util.BytesToUint32(bytes[NeedleIdSize+OffsetSize : NeedleIdSize+OffsetSize+SizeSize])
- return
-}
func (nm *baseNeedleMapper) appendToIndexFile(key NeedleId, offset Offset, size uint32) error {
bytes := needle_map.ToBytes(key, offset, size)
diff --git a/weed/storage/needle_map_leveldb.go b/weed/storage/needle_map_leveldb.go
index 602c90ddf..c2c6fcf19 100644
--- a/weed/storage/needle_map_leveldb.go
+++ b/weed/storage/needle_map_leveldb.go
@@ -2,10 +2,12 @@ package storage
import (
"fmt"
- "github.com/syndtr/goleveldb/leveldb/opt"
"os"
"path/filepath"
+ "github.com/chrislusf/seaweedfs/weed/storage/idx"
+ "github.com/syndtr/goleveldb/leveldb/opt"
+
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
@@ -64,7 +66,7 @@ func generateLevelDbFile(dbFileName string, indexFile *os.File) error {
return err
}
defer db.Close()
- return WalkIndexFile(indexFile, func(key NeedleId, offset Offset, size uint32) error {
+ return idx.WalkIndexFile(indexFile, func(key NeedleId, offset Offset, size uint32) error {
if !offset.IsZero() && size != TombstoneFileSize {
levelDbWrite(db, key, offset, size)
} else {
diff --git a/weed/storage/needle_map_memory.go b/weed/storage/needle_map_memory.go
index bda4a5c8f..ee639a7e6 100644
--- a/weed/storage/needle_map_memory.go
+++ b/weed/storage/needle_map_memory.go
@@ -1,10 +1,10 @@
package storage
import (
- "io"
"os"
"github.com/chrislusf/seaweedfs/weed/glog"
+ "github.com/chrislusf/seaweedfs/weed/storage/idx"
"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
)
@@ -30,10 +30,6 @@ func NewBtreeNeedleMap(file *os.File) *NeedleMap {
return nm
}
-const (
- RowsToRead = 1024
-)
-
func LoadCompactNeedleMap(file *os.File) (*NeedleMap, error) {
nm := NewCompactNeedleMap(file)
return doLoading(file, nm)
@@ -45,7 +41,7 @@ func LoadBtreeNeedleMap(file *os.File) (*NeedleMap, error) {
}
func doLoading(file *os.File, nm *NeedleMap) (*NeedleMap, error) {
- e := WalkIndexFile(file, func(key NeedleId, offset Offset, size uint32) error {
+ e := idx.WalkIndexFile(file, func(key NeedleId, offset Offset, size uint32) error {
nm.MaybeSetMaxFileKey(key)
if !offset.IsZero() && size != TombstoneFileSize {
nm.FileCounter++
@@ -68,38 +64,6 @@ func doLoading(file *os.File, nm *NeedleMap) (*NeedleMap, error) {
return nm, e
}
-// walks through the index file, calls fn function with each key, offset, size
-// stops with the error returned by the fn function
-func WalkIndexFile(r *os.File, fn func(key NeedleId, offset Offset, size uint32) error) error {
- var readerOffset int64
- bytes := make([]byte, NeedleMapEntrySize*RowsToRead)
- count, e := r.ReadAt(bytes, readerOffset)
- glog.V(3).Infoln("file", r.Name(), "readerOffset", readerOffset, "count", count, "e", e)
- readerOffset += int64(count)
- var (
- key NeedleId
- offset Offset
- size uint32
- i int
- )
-
- for count > 0 && e == nil || e == io.EOF {
- for i = 0; i+NeedleMapEntrySize <= count; i += NeedleMapEntrySize {
- key, offset, size = IdxFileEntry(bytes[i : i+NeedleMapEntrySize])
- if e = fn(key, offset, size); e != nil {
- return e
- }
- }
- if e == io.EOF {
- return nil
- }
- count, e = r.ReadAt(bytes, readerOffset)
- glog.V(3).Infoln("file", r.Name(), "readerOffset", readerOffset, "count", count, "e", e)
- readerOffset += int64(count)
- }
- return e
-}
-
func (nm *NeedleMap) Put(key NeedleId, offset Offset, size uint32) error {
_, oldSize := nm.m.Set(NeedleId(key), offset, size)
nm.logPut(key, oldSize, size)
diff --git a/weed/storage/needle_map_metric.go b/weed/storage/needle_map_metric.go
index 55b877db6..f7372105f 100644
--- a/weed/storage/needle_map_metric.go
+++ b/weed/storage/needle_map_metric.go
@@ -5,6 +5,7 @@ import (
"os"
"sync/atomic"
+ "github.com/chrislusf/seaweedfs/weed/storage/idx"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/willf/bloom"
)
@@ -119,7 +120,7 @@ func reverseWalkIndexFile(r *os.File, initFn func(entryCount int64), fn func(key
return e
}
for i := int(nextBatchSize) - 1; i >= 0; i-- {
- key, offset, size := IdxFileEntry(bytes[i*NeedleMapEntrySize : i*NeedleMapEntrySize+NeedleMapEntrySize])
+ key, offset, size := idx.IdxFileEntry(bytes[i*NeedleMapEntrySize : i*NeedleMapEntrySize+NeedleMapEntrySize])
if e = fn(key, offset, size); e != nil {
return e
}
diff --git a/weed/storage/store.go b/weed/storage/store.go
index bde6d872a..d9c9b0e35 100644
--- a/weed/storage/store.go
+++ b/weed/storage/store.go
@@ -30,6 +30,8 @@ type Store struct {
NeedleMapType NeedleMapType
NewVolumesChan chan master_pb.VolumeShortInformationMessage
DeletedVolumesChan chan master_pb.VolumeShortInformationMessage
+ NewEcShardsChan chan master_pb.VolumeEcShardInformationMessage
+ DeletedEcShardsChan chan master_pb.VolumeEcShardInformationMessage
}
func (s *Store) String() (str string) {
@@ -47,6 +49,10 @@ func NewStore(port int, ip, publicUrl string, dirnames []string, maxVolumeCounts
}
s.NewVolumesChan = make(chan master_pb.VolumeShortInformationMessage, 3)
s.DeletedVolumesChan = make(chan master_pb.VolumeShortInformationMessage, 3)
+
+ s.NewEcShardsChan = make(chan master_pb.VolumeEcShardInformationMessage, 3)
+ s.DeletedEcShardsChan = make(chan master_pb.VolumeEcShardInformationMessage, 3)
+
return
}
func (s *Store) AddVolume(volumeId needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement string, ttlString string, preallocate int64) error {
@@ -186,6 +192,7 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat {
}
}
+
func (s *Store) Close() {
for _, location := range s.Locations {
location.Close()
diff --git a/weed/storage/store_ec.go b/weed/storage/store_ec.go
new file mode 100644
index 000000000..43d851faa
--- /dev/null
+++ b/weed/storage/store_ec.go
@@ -0,0 +1,21 @@
+package storage
+
+import (
+ "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+)
+
+func (s *Store) CollectErasureCodingHeartbeat() *master_pb.Heartbeat {
+ var ecShardMessages []*master_pb.VolumeEcShardInformationMessage
+ for _, location := range s.Locations {
+ location.ecShardsLock.RLock()
+ for _, ecShards := range location.ecShards {
+ ecShardMessages = append(ecShardMessages, ecShards.ToVolumeInformationMessage()...)
+ }
+ location.ecShardsLock.RUnlock()
+ }
+
+ return &master_pb.Heartbeat{
+ EcShards: ecShardMessages,
+ }
+
+}
diff --git a/weed/storage/volume_backup.go b/weed/storage/volume_backup.go
index b7ca3c023..f56c40019 100644
--- a/weed/storage/volume_backup.go
+++ b/weed/storage/volume_backup.go
@@ -8,6 +8,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
+ "github.com/chrislusf/seaweedfs/weed/storage/idx"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"google.golang.org/grpc"
@@ -142,7 +143,7 @@ func (v *Volume) locateLastAppendEntry() (Offset, error) {
if n != NeedleMapEntrySize {
return Offset{}, fmt.Errorf("file %s read error: %v", indexFile.Name(), e)
}
- _, offset, _ := IdxFileEntry(bytes)
+ _, offset, _ := idx.IdxFileEntry(bytes)
return offset, nil
}
@@ -230,7 +231,7 @@ func (v *Volume) readAppendAtNsForIndexEntry(indexFile *os.File, bytes []byte, m
if _, readErr := indexFile.ReadAt(bytes, m*NeedleMapEntrySize); readErr != nil && readErr != io.EOF {
return Offset{}, readErr
}
- _, offset, _ := IdxFileEntry(bytes)
+ _, offset, _ := idx.IdxFileEntry(bytes)
return offset, nil
}
diff --git a/weed/storage/volume_checking.go b/weed/storage/volume_checking.go
index 7bb998fa6..8f930546f 100644
--- a/weed/storage/volume_checking.go
+++ b/weed/storage/volume_checking.go
@@ -4,6 +4,7 @@ import (
"fmt"
"os"
+ "github.com/chrislusf/seaweedfs/weed/storage/idx"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
@@ -21,7 +22,7 @@ func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) (lastAppendAtNs uin
if lastIdxEntry, e = readIndexEntryAtOffset(indexFile, indexSize-NeedleMapEntrySize); e != nil {
return 0, fmt.Errorf("readLastIndexEntry %s failed: %v", indexFile.Name(), e)
}
- key, offset, size := IdxFileEntry(lastIdxEntry)
+ key, offset, size := idx.IdxFileEntry(lastIdxEntry)
if offset.IsZero() {
return 0, nil
}
diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go
index bbe17071e..f89f2cf0a 100644
--- a/weed/storage/volume_vacuum.go
+++ b/weed/storage/volume_vacuum.go
@@ -6,6 +6,7 @@ import (
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
+ idx2 "github.com/chrislusf/seaweedfs/weed/storage/idx"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
@@ -143,7 +144,7 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI
if IdxEntry, err = readIndexEntryAtOffset(oldIdxFile, idxOffset); err != nil {
return fmt.Errorf("readIndexEntry %s at offset %d failed: %v", oldIdxFileName, idxOffset, err)
}
- key, offset, size := IdxFileEntry(IdxEntry)
+ key, offset, size := idx2.IdxFileEntry(IdxEntry)
glog.V(4).Infof("key %d offset %d size %d", key, offset, size)
if _, found := incrementedHasUpdatedIndexEntry[key]; !found {
incrementedHasUpdatedIndexEntry[key] = keyField{
@@ -329,7 +330,7 @@ func (v *Volume) copyDataBasedOnIndexFile(dstName, idxName string) (err error) {
dst.Write(v.SuperBlock.Bytes())
newOffset := int64(v.SuperBlock.BlockSize())
- WalkIndexFile(oldIndexFile, func(key NeedleId, offset Offset, size uint32) error {
+ idx2.WalkIndexFile(oldIndexFile, func(key NeedleId, offset Offset, size uint32) error {
if offset.IsZero() || size == TombstoneFileSize {
return nil
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-21 03:18:47 +0000