aboutsummaryrefslogtreecommitdiff
path: root/weed
diff options
context:
space:
mode:
Diffstat (limited to 'weed')
-rw-r--r--weed/command/filer_meta_tail.go7
-rw-r--r--weed/filer/entry_codec.go2
-rw-r--r--weed/filer/filer_conf.go12
-rw-r--r--weed/filer/s3iam_conf.go23
-rw-r--r--weed/mq/broker/broker_segment_serde.go4
-rw-r--r--weed/notification/aws_sqs/aws_sqs_pub.go7
-rw-r--r--weed/pb/proto_read_write_test.go2
-rw-r--r--weed/replication/sub/notification_aws_sqs.go2
-rw-r--r--weed/s3api/auth_credentials_test.go2
-rw-r--r--weed/shell/command_fs_meta_cat.go20
-rw-r--r--weed/shell/command_remote_configure.go10
-rw-r--r--weed/shell/command_remote_mount.go13
-rw-r--r--weed/storage/volume_info/volume_info.go15
-rw-r--r--weed/util/skiplist/Makefile2
-rw-r--r--weed/util/skiplist/skiplist.pb.go11
15 files changed, 47 insertions, 85 deletions
diff --git a/weed/command/filer_meta_tail.go b/weed/command/filer_meta_tail.go
index a433d1b07..58a496ff4 100644
--- a/weed/command/filer_meta_tail.go
+++ b/weed/command/filer_meta_tail.go
@@ -2,8 +2,8 @@ package command
import (
"fmt"
+ "github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/pb"
- "google.golang.org/protobuf/jsonpb"
"os"
"path/filepath"
"strings"
@@ -88,11 +88,8 @@ func runFilerMetaTail(cmd *Command, args []string) bool {
return false
}
- jsonpbMarshaler := jsonpb.Marshaler{
- EmitDefaults: false,
- }
eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error {
- jsonpbMarshaler.Marshal(os.Stdout, resp)
+ filer.ProtoToText(os.Stdout, resp)
fmt.Fprintln(os.Stdout)
return nil
}
diff --git a/weed/filer/entry_codec.go b/weed/filer/entry_codec.go
index 786ecea49..ce9c0484b 100644
--- a/weed/filer/entry_codec.go
+++ b/weed/filer/entry_codec.go
@@ -21,7 +21,7 @@ func (entry *Entry) DecodeAttributesAndChunks(blob []byte) error {
message := &filer_pb.Entry{}
- if err := proto.UnmarshalMerge(blob, message); err != nil {
+ if err := proto.Unmarshal(blob, message); err != nil {
return fmt.Errorf("decoding value blob for %s: %v", entry.FullPath, err)
}
diff --git a/weed/filer/filer_conf.go b/weed/filer/filer_conf.go
index 59abaa5ab..30161cfaf 100644
--- a/weed/filer/filer_conf.go
+++ b/weed/filer/filer_conf.go
@@ -13,7 +13,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/viant/ptrie"
- "google.golang.org/protobuf/jsonpb"
+ jsonpb "google.golang.org/protobuf/encoding/protojson"
)
const (
@@ -93,7 +93,7 @@ func (fc *FilerConf) loadFromChunks(filer *Filer, content []byte, chunks []*file
func (fc *FilerConf) LoadFromBytes(data []byte) (err error) {
conf := &filer_pb.FilerConf{}
- if err := jsonpb.Unmarshal(bytes.NewReader(data), conf); err != nil {
+ if err := jsonpb.Unmarshal(data, conf); err != nil {
return err
}
@@ -181,11 +181,5 @@ func (fc *FilerConf) ToProto() *filer_pb.FilerConf {
}
func (fc *FilerConf) ToText(writer io.Writer) error {
-
- m := jsonpb.Marshaler{
- EmitDefaults: false,
- Indent: " ",
- }
-
- return m.Marshal(writer, fc.ToProto())
+ return ProtoToText(writer, fc.ToProto())
}
diff --git a/weed/filer/s3iam_conf.go b/weed/filer/s3iam_conf.go
index 6240058a4..24ed46be5 100644
--- a/weed/filer/s3iam_conf.go
+++ b/weed/filer/s3iam_conf.go
@@ -1,17 +1,16 @@
package filer
import (
- "bytes"
"fmt"
"io"
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
- "google.golang.org/protobuf/jsonpb"
+ jsonpb "google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/proto"
)
func ParseS3ConfigurationFromBytes[T proto.Message](content []byte, config T) error {
- if err := jsonpb.Unmarshal(bytes.NewBuffer(content), config); err != nil {
+ if err := jsonpb.Unmarshal(content, config); err != nil {
return err
}
return nil
@@ -19,12 +18,22 @@ func ParseS3ConfigurationFromBytes[T proto.Message](content []byte, config T) er
func ProtoToText(writer io.Writer, config proto.Message) error {
- m := jsonpb.Marshaler{
- EmitDefaults: false,
- Indent: " ",
+ m := jsonpb.MarshalOptions{
+ EmitUnpopulated: true,
+ Indent: " ",
}
- return m.Marshal(writer, config)
+ text, marshalErr := m.Marshal(config)
+ if marshalErr != nil {
+ return fmt.Errorf("marshal proto message: %v", marshalErr)
+ }
+
+ _, writeErr := writer.Write(text)
+ if writeErr != nil {
+ return fmt.Errorf("fail to write proto message: %v", writeErr)
+ }
+
+ return writeErr
}
// CheckDuplicateAccessKey returns an error message when s3cfg has duplicate access keys
diff --git a/weed/mq/broker/broker_segment_serde.go b/weed/mq/broker/broker_segment_serde.go
index 44bc1b7d3..a8b1d05bc 100644
--- a/weed/mq/broker/broker_segment_serde.go
+++ b/weed/mq/broker/broker_segment_serde.go
@@ -8,7 +8,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
- "google.golang.org/protobuf/jsonpb"
+ jsonpb "google.golang.org/protobuf/encoding/protojson"
"time"
)
@@ -60,7 +60,7 @@ func (broker *MessageQueueBroker) readSegmentOnFiler(segment *mq.Segment) (info
// parse into filer conf object
info = &mq_pb.SegmentInfo{}
- if err = jsonpb.Unmarshal(bytes.NewReader(data), info); err != nil {
+ if err = jsonpb.Unmarshal(data, info); err != nil {
return err
}
found = true
diff --git a/weed/notification/aws_sqs/aws_sqs_pub.go b/weed/notification/aws_sqs/aws_sqs_pub.go
index f647134c7..c9e674257 100644
--- a/weed/notification/aws_sqs/aws_sqs_pub.go
+++ b/weed/notification/aws_sqs/aws_sqs_pub.go
@@ -70,7 +70,10 @@ func (k *AwsSqsPub) initialize(awsAccessKeyId, awsSecretAccessKey, region, queue
func (k *AwsSqsPub) SendMessage(key string, message proto.Message) (err error) {
- text := proto.MarshalTextString(message)
+ text, err := proto.Marshal(message)
+ if err != nil {
+ return fmt.Errorf("send message marshal %+v: %v", message, err)
+ }
_, err = k.svc.SendMessage(&sqs.SendMessageInput{
DelaySeconds: aws.Int64(10),
@@ -80,7 +83,7 @@ func (k *AwsSqsPub) SendMessage(key string, message proto.Message) (err error) {
StringValue: aws.String(key),
},
},
- MessageBody: aws.String(text),
+ MessageBody: aws.String(string(text)),
QueueUrl: &k.queueUrl,
})
diff --git a/weed/pb/proto_read_write_test.go b/weed/pb/proto_read_write_test.go
index f4751c3bb..ac3b960ee 100644
--- a/weed/pb/proto_read_write_test.go
+++ b/weed/pb/proto_read_write_test.go
@@ -5,7 +5,7 @@ import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
- "google.golang.org/protobuf/jsonpb"
+ jsonpb "google.golang.org/protobuf/encoding/protojson"
)
func TestJsonpMarshalUnmarshal(t *testing.T) {
diff --git a/weed/replication/sub/notification_aws_sqs.go b/weed/replication/sub/notification_aws_sqs.go
index 146f42c72..0201d4f9c 100644
--- a/weed/replication/sub/notification_aws_sqs.go
+++ b/weed/replication/sub/notification_aws_sqs.go
@@ -98,7 +98,7 @@ func (k *AwsSqsInput) ReceiveMessage() (key string, message *filer_pb.EventNotif
key = *keyValue.StringValue
text := *result.Messages[0].Body
message = &filer_pb.EventNotification{}
- err = proto.UnmarshalText(text, message)
+ err = proto.Unmarshal([]byte(text), message)
// delete the message
_, err = k.svc.DeleteMessage(&sqs.DeleteMessageInput{
diff --git a/weed/s3api/auth_credentials_test.go b/weed/s3api/auth_credentials_test.go
index dacbc775d..09692eb81 100644
--- a/weed/s3api/auth_credentials_test.go
+++ b/weed/s3api/auth_credentials_test.go
@@ -5,7 +5,7 @@ import (
"github.com/stretchr/testify/assert"
"testing"
- "google.golang.org/protobuf/jsonpb"
+ jsonpb "google.golang.org/protobuf/encoding/protojson"
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
)
diff --git a/weed/shell/command_fs_meta_cat.go b/weed/shell/command_fs_meta_cat.go
index 4f3b3551e..3ae106415 100644
--- a/weed/shell/command_fs_meta_cat.go
+++ b/weed/shell/command_fs_meta_cat.go
@@ -2,8 +2,7 @@ package shell
import (
"fmt"
- "golang.org/x/exp/slices"
- "google.golang.org/protobuf/jsonpb"
+ "github.com/seaweedfs/seaweedfs/weed/filer"
"google.golang.org/protobuf/proto"
"io"
@@ -50,22 +49,7 @@ func (c *commandFsMetaCat) Do(args []string, commandEnv *CommandEnv, writer io.W
return err
}
- m := jsonpb.Marshaler{
- EmitDefaults: true,
- Indent: " ",
- }
- slices.SortFunc(respLookupEntry.Entry.Chunks, func(a, b *filer_pb.FileChunk) bool {
- if a.Offset == b.Offset {
- return a.Mtime < b.Mtime
- }
- return a.Offset < b.Offset
- })
- text, marshalErr := m.MarshalToString(respLookupEntry.Entry)
- if marshalErr != nil {
- return fmt.Errorf("marshal meta: %v", marshalErr)
- }
-
- fmt.Fprintf(writer, "%s\n", text)
+ filer.ProtoToText(writer, respLookupEntry.Entry)
bytes, _ := proto.Marshal(respLookupEntry.Entry)
gzippedBytes, _ := util.GzipData(bytes)
diff --git a/weed/shell/command_remote_configure.go b/weed/shell/command_remote_configure.go
index 306858271..4491dc456 100644
--- a/weed/shell/command_remote_configure.go
+++ b/weed/shell/command_remote_configure.go
@@ -9,7 +9,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
"github.com/seaweedfs/seaweedfs/weed/remote_storage"
"github.com/seaweedfs/seaweedfs/weed/util"
- "google.golang.org/protobuf/jsonpb"
"google.golang.org/protobuf/proto"
"io"
"regexp"
@@ -159,15 +158,8 @@ func (c *commandRemoteConfigure) listExistingRemoteStorages(commandEnv *CommandE
conf.TencentSecretKey = strings.Repeat("*", len(conf.TencentSecretKey))
conf.WasabiSecretKey = strings.Repeat("*", len(conf.WasabiSecretKey))
- m := jsonpb.Marshaler{
- EmitDefaults: false,
- Indent: " ",
- }
-
- err := m.Marshal(writer, conf)
- fmt.Fprintln(writer)
+ return filer.ProtoToText(writer, conf)
- return err
})
}
diff --git a/weed/shell/command_remote_mount.go b/weed/shell/command_remote_mount.go
index ef837a821..0f74729f1 100644
--- a/weed/shell/command_remote_mount.go
+++ b/weed/shell/command_remote_mount.go
@@ -9,7 +9,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
"github.com/seaweedfs/seaweedfs/weed/remote_storage"
"github.com/seaweedfs/seaweedfs/weed/util"
- "google.golang.org/protobuf/jsonpb"
"google.golang.org/protobuf/proto"
"io"
"os"
@@ -101,17 +100,7 @@ func listExistingRemoteStorageMounts(commandEnv *CommandEnv, writer io.Writer) (
}
func jsonPrintln(writer io.Writer, message proto.Message) error {
- if message == nil {
- return nil
- }
- m := jsonpb.Marshaler{
- EmitDefaults: false,
- Indent: " ",
- }
-
- err := m.Marshal(writer, message)
- fmt.Fprintln(writer)
- return err
+ return filer.ProtoToText(writer, message)
}
func syncMetadata(commandEnv *CommandEnv, writer io.Writer, dir string, nonEmpty bool, remoteConf *remote_pb.RemoteConf, remote *remote_pb.RemoteStorageLocation) error {
diff --git a/weed/storage/volume_info/volume_info.go b/weed/storage/volume_info/volume_info.go
index 8002e9c19..7f8663be2 100644
--- a/weed/storage/volume_info/volume_info.go
+++ b/weed/storage/volume_info/volume_info.go
@@ -1,11 +1,10 @@
package volume_info
import (
- "bytes"
"fmt"
"os"
- "google.golang.org/protobuf/jsonpb"
+ jsonpb "google.golang.org/protobuf/encoding/protojson"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
@@ -44,7 +43,7 @@ func MaybeLoadVolumeInfo(fileName string) (volumeInfo *volume_server_pb.VolumeIn
}
glog.V(1).Infof("maybeLoadVolumeInfo Unmarshal volume info %v", fileName)
- if err = jsonpb.Unmarshal(bytes.NewReader(tierData), volumeInfo); err != nil {
+ if err = jsonpb.Unmarshal(tierData, volumeInfo); err != nil {
glog.Warningf("unmarshal error: %v", err)
err = fmt.Errorf("unmarshal error: %v", err)
return
@@ -65,17 +64,17 @@ func SaveVolumeInfo(fileName string, volumeInfo *volume_server_pb.VolumeInfo) er
return fmt.Errorf("%s not writable", fileName)
}
- m := jsonpb.Marshaler{
- EmitDefaults: true,
- Indent: " ",
+ m := jsonpb.MarshalOptions{
+ EmitUnpopulated: true,
+ Indent: " ",
}
- text, marshalErr := m.MarshalToString(volumeInfo)
+ text, marshalErr := m.Marshal(volumeInfo)
if marshalErr != nil {
return fmt.Errorf("marshal to %s: %v", fileName, marshalErr)
}
- writeErr := util.WriteFile(fileName, []byte(text), 0755)
+ writeErr := util.WriteFile(fileName, text, 0755)
if writeErr != nil {
return fmt.Errorf("fail to write %s : %v", fileName, writeErr)
}
diff --git a/weed/util/skiplist/Makefile b/weed/util/skiplist/Makefile
index af4afe639..7c4652cba 100644
--- a/weed/util/skiplist/Makefile
+++ b/weed/util/skiplist/Makefile
@@ -3,4 +3,4 @@ all: gen
.PHONY : gen
gen:
- protoc skiplist.proto --go_out=plugins=grpc:. --go_opt=paths=source_relative
+ protoc skiplist.proto --go_out=. --go-grpc_out=. --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
diff --git a/weed/util/skiplist/skiplist.pb.go b/weed/util/skiplist/skiplist.pb.go
index ef3827a65..de3efa513 100644
--- a/weed/util/skiplist/skiplist.pb.go
+++ b/weed/util/skiplist/skiplist.pb.go
@@ -1,13 +1,12 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
-// protoc-gen-go v1.25.0
-// protoc v3.12.3
+// protoc-gen-go v1.26.0
+// protoc v3.17.3
// source: skiplist.proto
package skiplist
import (
- proto "google.golang.org/protobuf/proto"
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
@@ -21,10 +20,6 @@ const (
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
-// This is a compile-time assertion that a sufficiently up-to-date version
-// of the legacy proto package is being used.
-const _ = proto.ProtoPackageIsVersion4
-
type SkipListProto struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -325,7 +320,7 @@ var file_skiplist_proto_rawDesc = []byte{
0x22, 0x25, 0x0a, 0x0d, 0x4e, 0x61, 0x6d, 0x65, 0x42, 0x61, 0x74, 0x63, 0x68, 0x44, 0x61, 0x74,
0x61, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c,
0x52, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x42, 0x33, 0x5a, 0x31, 0x67, 0x69, 0x74, 0x68, 0x75,
- 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f,
+ 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f,
0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x75,
0x74, 0x69, 0x6c, 0x2f, 0x73, 0x6b, 0x69, 0x70, 0x6c, 0x69, 0x73, 0x74, 0x62, 0x06, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x33,
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-21 11:56:29 +0000