diff options
| author | Chris Lu <chrislusf@users.noreply.github.com> | 2023-12-11 12:05:54 -0800 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-12-11 12:05:54 -0800 |
| commit | 580940bf8214ac467694a09436ffc6d97066b97c (patch) | |
| tree | abcc275ebd0088b817f9155b8c08f1bc3d3a0fae /weed/mq/broker/broker_segment_serde.go | |
| parent | 8784553501f6569cc2419769f18f09099cb7d30c (diff) | |
| download | seaweedfs-580940bf8214ac467694a09436ffc6d97066b97c.tar.xz seaweedfs-580940bf8214ac467694a09436ffc6d97066b97c.zip | |
Merge accumulated changes related to message queue (#5098)
* balance partitions on brokers
* prepare topic partition first and then publish, move partition
* purge unused APIs
* clean up
* adjust logs
* add BalanceTopics() grpc API
* configure topic
* configure topic command
* refactor
* repair missing partitions
* sequence of operations to ensure ordering
* proto to close publishers and consumers
* rename file
* topic partition versioned by unixTimeNs
* create local topic partition
* close publishers
* randomize the client name
* wait until no publishers
* logs
* close stop publisher channel
* send last ack
* comments
* comment
* comments
* support list of brokers
* add cli options
* Update .gitignore
* logs
* return io.eof directly
* refactor
* optionally create topic
* refactoring
* detect consumer disconnection
* sub client wait for more messages
* subscribe by time stamp
* rename
* rename to sub_balancer
* rename
* adjust comments
* rename
* fix compilation
* rename
* rename
* SubscriberToSubCoordinator
* sticky rebalance
* go fmt
* add tests
* balance partitions on brokers
* prepare topic partition first and then publish, move partition
* purge unused APIs
* clean up
* adjust logs
* add BalanceTopics() grpc API
* configure topic
* configure topic command
* refactor
* repair missing partitions
* sequence of operations to ensure ordering
* proto to close publishers and consumers
* rename file
* topic partition versioned by unixTimeNs
* create local topic partition
* close publishers
* randomize the client name
* wait until no publishers
* logs
* close stop publisher channel
* send last ack
* comments
* comment
* comments
* support list of brokers
* add cli options
* Update .gitignore
* logs
* return io.eof directly
* refactor
* optionally create topic
* refactoring
* detect consumer disconnection
* sub client wait for more messages
* subscribe by time stamp
* rename
* rename to sub_balancer
* rename
* adjust comments
* rename
* fix compilation
* rename
* rename
* SubscriberToSubCoordinator
* sticky rebalance
* go fmt
* add tests
* tracking topic=>broker
* merge
* comment
Diffstat (limited to 'weed/mq/broker/broker_segment_serde.go')
| -rw-r--r-- | weed/mq/broker/broker_segment_serde.go | 89 |
1 files changed, 0 insertions, 89 deletions
diff --git a/weed/mq/broker/broker_segment_serde.go b/weed/mq/broker/broker_segment_serde.go deleted file mode 100644 index bb9aecc0b..000000000 --- a/weed/mq/broker/broker_segment_serde.go +++ /dev/null @@ -1,89 +0,0 @@ -package broker - -import ( - "bytes" - "fmt" - "github.com/seaweedfs/seaweedfs/weed/filer" - "github.com/seaweedfs/seaweedfs/weed/mq/topic" - "github.com/seaweedfs/seaweedfs/weed/pb" - "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" - "github.com/seaweedfs/seaweedfs/weed/pb/mq_pb" - jsonpb "google.golang.org/protobuf/encoding/protojson" - "time" -) - -func (broker *MessageQueueBroker) checkSegmentOnFiler(segment *topic.Segment) (brokers []pb.ServerAddress, err error) { - info, found, err := broker.readSegmentInfoOnFiler(segment) - if err != nil { - return - } - if !found { - return - } - for _, b := range info.Brokers { - brokers = append(brokers, pb.ServerAddress(b)) - } - - return -} - -func (broker *MessageQueueBroker) saveSegmentBrokersOnFiler(segment *topic.Segment, brokers []pb.ServerAddress) (err error) { - var nodes []string - for _, b := range brokers { - nodes = append(nodes, string(b)) - } - broker.saveSegmentInfoToFiler(segment, &mq_pb.SegmentInfo{ - Segment: segment.ToPbSegment(), - StartTsNs: time.Now().UnixNano(), - Brokers: nodes, - StopTsNs: 0, - PreviousSegments: nil, - NextSegments: nil, - }) - return -} - -func (broker *MessageQueueBroker) readSegmentInfoOnFiler(segment *topic.Segment) (info *mq_pb.SegmentInfo, found bool, err error) { - dir, name := segment.DirAndName() - - found, err = filer_pb.Exists(broker, dir, name, false) - if !found || err != nil { - return - } - - err = pb.WithFilerClient(false, 0, broker.GetFiler(), broker.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { - // read filer conf first - data, err := filer.ReadInsideFiler(client, dir, name) - if err != nil { - return fmt.Errorf("ReadEntry: %v", err) - } - - // parse into filer conf object - info = &mq_pb.SegmentInfo{} - if err = jsonpb.Unmarshal(data, info); err != nil { - return err - } - found = true - return nil - }) - - return -} - -func (broker *MessageQueueBroker) saveSegmentInfoToFiler(segment *topic.Segment, info *mq_pb.SegmentInfo) (err error) { - dir, name := segment.DirAndName() - - var buf bytes.Buffer - filer.ProtoToText(&buf, info) - - err = pb.WithFilerClient(false, 0, broker.GetFiler(), broker.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { - // read filer conf first - err := filer.SaveInsideFiler(client, dir, name, buf.Bytes()) - if err != nil { - return fmt.Errorf("save segment info: %v", err) - } - return nil - }) - - return -} |