aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--other/java/client/src/main/proto/filer.proto4
-rw-r--r--weed/filesys/dir.go74
-rw-r--r--weed/pb/filer.proto4
-rw-r--r--weed/pb/filer_pb/filer.pb.go290
-rw-r--r--weed/s3api/filer_util.go17
-rw-r--r--weed/s3api/s3api_objects_list_handlers.go17
-rw-r--r--weed/server/filer_grpc_server.go29
-rw-r--r--weed/server/webdav_server.go2
-rw-r--r--weed/shell/command_fs_du.go96
-rw-r--r--weed/shell/command_fs_ls.go99
-rw-r--r--weed/shell/command_fs_meta_notify.go45
-rw-r--r--weed/shell/command_fs_meta_save.go144
-rw-r--r--weed/shell/command_fs_tree.go87
-rw-r--r--weed/shell/commands.go24
14 files changed, 421 insertions, 511 deletions
diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto
index 18ccca44f..ef847cbe7 100644
--- a/other/java/client/src/main/proto/filer.proto
+++ b/other/java/client/src/main/proto/filer.proto
@@ -12,7 +12,7 @@ service SeaweedFiler {
rpc LookupDirectoryEntry (LookupDirectoryEntryRequest) returns (LookupDirectoryEntryResponse) {
}
- rpc ListEntries (ListEntriesRequest) returns (ListEntriesResponse) {
+ rpc ListEntries (ListEntriesRequest) returns (stream ListEntriesResponse) {
}
rpc CreateEntry (CreateEntryRequest) returns (CreateEntryResponse) {
@@ -64,7 +64,7 @@ message ListEntriesRequest {
}
message ListEntriesResponse {
- repeated Entry entries = 1;
+ Entry entry = 1;
}
message Entry {
diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go
index abbc9288e..fd5356ae0 100644
--- a/weed/filesys/dir.go
+++ b/weed/filesys/dir.go
@@ -212,51 +212,23 @@ func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.
func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
- err = dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
-
- remaining := dir.wfs.option.DirListingLimit
-
- lastEntryName := ""
-
- for remaining >= 0 {
-
- request := &filer_pb.ListEntriesRequest{
- Directory: dir.Path,
- StartFromFileName: lastEntryName,
- Limit: filer2.PaginationSize,
- }
-
- glog.V(4).Infof("read directory: %v", request)
- resp, err := client.ListEntries(ctx, request)
- if err != nil {
- glog.V(0).Infof("list %s: %v", dir.Path, err)
- return fuse.EIO
- }
-
- cacheTtl := estimatedCacheTtl(len(resp.Entries))
-
- for _, entry := range resp.Entries {
- if entry.IsDirectory {
- dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_Dir}
- ret = append(ret, dirent)
- } else {
- dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_File}
- ret = append(ret, dirent)
- }
- dir.wfs.listDirectoryEntriesCache.Set(path.Join(dir.Path, entry.Name), entry, cacheTtl)
- lastEntryName = entry.Name
- }
-
- remaining -= len(resp.Entries)
-
- if len(resp.Entries) < filer2.PaginationSize {
- break
- }
+ cacheTtl := 3 * time.Second
+ readErr := filer2.ReadDirAllEntries(ctx, dir.wfs, dir.Path, "", func(entry *filer_pb.Entry, isLast bool) {
+ if entry.IsDirectory {
+ dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_Dir}
+ ret = append(ret, dirent)
+ } else {
+ dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_File}
+ ret = append(ret, dirent)
}
-
- return nil
+ cacheTtl = cacheTtl + 2 * time.Millisecond
+ dir.wfs.listDirectoryEntriesCache.Set(path.Join(dir.Path, entry.Name), entry, cacheTtl)
})
+ if readErr != nil {
+ glog.V(0).Infof("list %s: %v", dir.Path, err)
+ return ret, fuse.EIO
+ }
return ret, err
}
@@ -373,21 +345,3 @@ func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fus
})
}
-
-func estimatedCacheTtl(numEntries int) time.Duration {
- if numEntries < 100 {
- // 30 ms per entry
- return 3 * time.Second
- }
- if numEntries < 1000 {
- // 10 ms per entry
- return 10 * time.Second
- }
- if numEntries < 10000 {
- // 10 ms per entry
- return 100 * time.Second
- }
-
- // 2 ms per entry
- return time.Duration(numEntries*2) * time.Millisecond
-}
diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto
index 18ccca44f..ef847cbe7 100644
--- a/weed/pb/filer.proto
+++ b/weed/pb/filer.proto
@@ -12,7 +12,7 @@ service SeaweedFiler {
rpc LookupDirectoryEntry (LookupDirectoryEntryRequest) returns (LookupDirectoryEntryResponse) {
}
- rpc ListEntries (ListEntriesRequest) returns (ListEntriesResponse) {
+ rpc ListEntries (ListEntriesRequest) returns (stream ListEntriesResponse) {
}
rpc CreateEntry (CreateEntryRequest) returns (CreateEntryResponse) {
@@ -64,7 +64,7 @@ message ListEntriesRequest {
}
message ListEntriesResponse {
- repeated Entry entries = 1;
+ Entry entry = 1;
}
message Entry {
diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go
index 89541d6f3..c8214aa94 100644
--- a/weed/pb/filer_pb/filer.pb.go
+++ b/weed/pb/filer_pb/filer.pb.go
@@ -151,7 +151,7 @@ func (m *ListEntriesRequest) GetLimit() uint32 {
}
type ListEntriesResponse struct {
- Entries []*Entry `protobuf:"bytes,1,rep,name=entries" json:"entries,omitempty"`
+ Entry *Entry `protobuf:"bytes,1,opt,name=entry" json:"entry,omitempty"`
}
func (m *ListEntriesResponse) Reset() { *m = ListEntriesResponse{} }
@@ -159,9 +159,9 @@ func (m *ListEntriesResponse) String() string { return proto.CompactT
func (*ListEntriesResponse) ProtoMessage() {}
func (*ListEntriesResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} }
-func (m *ListEntriesResponse) GetEntries() []*Entry {
+func (m *ListEntriesResponse) GetEntry() *Entry {
if m != nil {
- return m.Entries
+ return m.Entry
}
return nil
}
@@ -1036,7 +1036,7 @@ const _ = grpc.SupportPackageIsVersion4
type SeaweedFilerClient interface {
LookupDirectoryEntry(ctx context.Context, in *LookupDirectoryEntryRequest, opts ...grpc.CallOption) (*LookupDirectoryEntryResponse, error)
- ListEntries(ctx context.Context, in *ListEntriesRequest, opts ...grpc.CallOption) (*ListEntriesResponse, error)
+ ListEntries(ctx context.Context, in *ListEntriesRequest, opts ...grpc.CallOption) (SeaweedFiler_ListEntriesClient, error)
CreateEntry(ctx context.Context, in *CreateEntryRequest, opts ...grpc.CallOption) (*CreateEntryResponse, error)
UpdateEntry(ctx context.Context, in *UpdateEntryRequest, opts ...grpc.CallOption) (*UpdateEntryResponse, error)
DeleteEntry(ctx context.Context, in *DeleteEntryRequest, opts ...grpc.CallOption) (*DeleteEntryResponse, error)
@@ -1065,13 +1065,36 @@ func (c *seaweedFilerClient) LookupDirectoryEntry(ctx context.Context, in *Looku
return out, nil
}
-func (c *seaweedFilerClient) ListEntries(ctx context.Context, in *ListEntriesRequest, opts ...grpc.CallOption) (*ListEntriesResponse, error) {
- out := new(ListEntriesResponse)
- err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/ListEntries", in, out, c.cc, opts...)
+func (c *seaweedFilerClient) ListEntries(ctx context.Context, in *ListEntriesRequest, opts ...grpc.CallOption) (SeaweedFiler_ListEntriesClient, error) {
+ stream, err := grpc.NewClientStream(ctx, &_SeaweedFiler_serviceDesc.Streams[0], c.cc, "/filer_pb.SeaweedFiler/ListEntries", opts...)
if err != nil {
return nil, err
}
- return out, nil
+ x := &seaweedFilerListEntriesClient{stream}
+ if err := x.ClientStream.SendMsg(in); err != nil {
+ return nil, err
+ }
+ if err := x.ClientStream.CloseSend(); err != nil {
+ return nil, err
+ }
+ return x, nil
+}
+
+type SeaweedFiler_ListEntriesClient interface {
+ Recv() (*ListEntriesResponse, error)
+ grpc.ClientStream
+}
+
+type seaweedFilerListEntriesClient struct {
+ grpc.ClientStream
+}
+
+func (x *seaweedFilerListEntriesClient) Recv() (*ListEntriesResponse, error) {
+ m := new(ListEntriesResponse)
+ if err := x.ClientStream.RecvMsg(m); err != nil {
+ return nil, err
+ }
+ return m, nil
}
func (c *seaweedFilerClient) CreateEntry(ctx context.Context, in *CreateEntryRequest, opts ...grpc.CallOption) (*CreateEntryResponse, error) {
@@ -1159,7 +1182,7 @@ func (c *seaweedFilerClient) GetFilerConfiguration(ctx context.Context, in *GetF
type SeaweedFilerServer interface {
LookupDirectoryEntry(context.Context, *LookupDirectoryEntryRequest) (*LookupDirectoryEntryResponse, error)
- ListEntries(context.Context, *ListEntriesRequest) (*ListEntriesResponse, error)
+ ListEntries(*ListEntriesRequest, SeaweedFiler_ListEntriesServer) error
CreateEntry(context.Context, *CreateEntryRequest) (*CreateEntryResponse, error)
UpdateEntry(context.Context, *UpdateEntryRequest) (*UpdateEntryResponse, error)
DeleteEntry(context.Context, *DeleteEntryRequest) (*DeleteEntryResponse, error)
@@ -1193,22 +1216,25 @@ func _SeaweedFiler_LookupDirectoryEntry_Handler(srv interface{}, ctx context.Con
return interceptor(ctx, in, info, handler)
}
-func _SeaweedFiler_ListEntries_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
- in := new(ListEntriesRequest)
- if err := dec(in); err != nil {
- return nil, err
- }
- if interceptor == nil {
- return srv.(SeaweedFilerServer).ListEntries(ctx, in)
+func _SeaweedFiler_ListEntries_Handler(srv interface{}, stream grpc.ServerStream) error {
+ m := new(ListEntriesRequest)
+ if err := stream.RecvMsg(m); err != nil {
+ return err
}
- info := &grpc.UnaryServerInfo{
- Server: srv,
- FullMethod: "/filer_pb.SeaweedFiler/ListEntries",
- }
- handler := func(ctx context.Context, req interface{}) (interface{}, error) {
- return srv.(SeaweedFilerServer).ListEntries(ctx, req.(*ListEntriesRequest))
- }
- return interceptor(ctx, in, info, handler)
+ return srv.(SeaweedFilerServer).ListEntries(m, &seaweedFilerListEntriesServer{stream})
+}
+
+type SeaweedFiler_ListEntriesServer interface {
+ Send(*ListEntriesResponse) error
+ grpc.ServerStream
+}
+
+type seaweedFilerListEntriesServer struct {
+ grpc.ServerStream
+}
+
+func (x *seaweedFilerListEntriesServer) Send(m *ListEntriesResponse) error {
+ return x.ServerStream.SendMsg(m)
}
func _SeaweedFiler_CreateEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
@@ -1382,10 +1408,6 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
Handler: _SeaweedFiler_LookupDirectoryEntry_Handler,
},
{
- MethodName: "ListEntries",
- Handler: _SeaweedFiler_ListEntries_Handler,
- },
- {
MethodName: "CreateEntry",
Handler: _SeaweedFiler_CreateEntry_Handler,
},
@@ -1422,113 +1444,119 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
Handler: _SeaweedFiler_GetFilerConfiguration_Handler,
},
},
- Streams: []grpc.StreamDesc{},
+ Streams: []grpc.StreamDesc{
+ {
+ StreamName: "ListEntries",
+ Handler: _SeaweedFiler_ListEntries_Handler,
+ ServerStreams: true,
+ },
+ },
Metadata: "filer.proto",
}
func init() { proto.RegisterFile("filer.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
- // 1608 bytes of a gzipped FileDescriptorProto
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0x49, 0x6f, 0xdc, 0x46,
- 0x16, 0x36, 0x7b, 0xe7, 0xeb, 0x6e, 0x5b, 0x2a, 0xc9, 0x36, 0xdd, 0x5a, 0x46, 0xa6, 0xc6, 0x1e,
- 0x19, 0x63, 0x68, 0x0c, 0x8f, 0x0f, 0xf6, 0x18, 0x03, 0xc4, 0xd6, 0x12, 0x08, 0x91, 0x17, 0x50,
- 0x76, 0x90, 0x20, 0x40, 0x08, 0x8a, 0xac, 0x6e, 0x55, 0x44, 0xb2, 0x3a, 0xc5, 0xa2, 0x24, 0xe7,
- 0x27, 0xe4, 0x98, 0x63, 0x80, 0x9c, 0xf3, 0x27, 0x82, 0x5c, 0x02, 0xff, 0x9d, 0x1c, 0x73, 0x0e,
- 0xaa, 0x8a, 0x64, 0x17, 0x9b, 0x2d, 0xc9, 0x41, 0xe0, 0x1b, 0xeb, 0x2d, 0xdf, 0x5b, 0xea, 0x2d,
- 0xd5, 0x0d, 0xdd, 0x21, 0x09, 0x31, 0xdb, 0x1c, 0x33, 0xca, 0x29, 0xea, 0xc8, 0x83, 0x3b, 0x3e,
- 0xb4, 0x5f, 0xc1, 0xd2, 0x3e, 0xa5, 0xc7, 0xe9, 0x78, 0x9b, 0x30, 0xec, 0x73, 0xca, 0xde, 0xed,
- 0xc4, 0x9c, 0xbd, 0x73, 0xf0, 0xb7, 0x29, 0x4e, 0x38, 0x5a, 0x06, 0x33, 0xc8, 0x19, 0x96, 0xb1,
- 0x66, 0x6c, 0x98, 0xce, 0x84, 0x80, 0x10, 0x34, 0x62, 0x2f, 0xc2, 0x56, 0x4d, 0x32, 0xe4, 0xb7,
- 0xbd, 0x03, 0xcb, 0xb3, 0x01, 0x93, 0x31, 0x8d, 0x13, 0x8c, 0xee, 0x40, 0x13, 0x0b, 0x82, 0x44,
- 0xeb, 0x3e, 0xbc, 0xb6, 0x99, 0xbb, 0xb2, 0xa9, 0xe4, 0x14, 0xd7, 0xfe, 0xd5, 0x00, 0xb4, 0x4f,
- 0x12, 0x2e, 0x88, 0x04, 0x27, 0x1f, 0xe6, 0xcf, 0x0d, 0x68, 0x8d, 0x19, 0x1e, 0x92, 0xb3, 0xcc,
- 0xa3, 0xec, 0x84, 0xee, 0xc3, 0x7c, 0xc2, 0x3d, 0xc6, 0x77, 0x19, 0x8d, 0x76, 0x49, 0x88, 0x5f,
- 0x0a, 0xa7, 0xeb, 0x52, 0xa4, 0xca, 0x40, 0x9b, 0x80, 0x48, 0xec, 0x87, 0x69, 0x42, 0x4e, 0xf0,
- 0x41, 0xce, 0xb5, 0x1a, 0x6b, 0xc6, 0x46, 0xc7, 0x99, 0xc1, 0x41, 0x8b, 0xd0, 0x0c, 0x49, 0x44,
- 0xb8, 0xd5, 0x5c, 0x33, 0x36, 0xfa, 0x8e, 0x3a, 0xd8, 0x9f, 0xc0, 0x42, 0xc9, 0xff, 0x2c, 0xfc,
- 0x7b, 0xd0, 0xc6, 0x8a, 0x64, 0x19, 0x6b, 0xf5, 0x59, 0x09, 0xc8, 0xf9, 0xf6, 0x4f, 0x35, 0x68,
- 0x4a, 0x52, 0x91, 0x67, 0x63, 0x92, 0x67, 0x74, 0x1b, 0x7a, 0x24, 0x71, 0x27, 0xc9, 0xa8, 0x49,
- 0xff, 0xba, 0x24, 0x29, 0xf2, 0x8e, 0xfe, 0x0d, 0x2d, 0xff, 0x28, 0x8d, 0x8f, 0x13, 0xab, 0x2e,
- 0x4d, 0x2d, 0x4c, 0x4c, 0x89, 0x60, 0xb7, 0x04, 0xcf, 0xc9, 0x44, 0xd0, 0x63, 0x00, 0x8f, 0x73,
- 0x46, 0x0e, 0x53, 0x8e, 0x13, 0x19, 0x6d, 0xf7, 0xa1, 0xa5, 0x29, 0xa4, 0x09, 0x7e, 0x56, 0xf0,
- 0x1d, 0x4d, 0x16, 0x3d, 0x81, 0x0e, 0x3e, 0xe3, 0x38, 0x0e, 0x70, 0x60, 0x35, 0xa5, 0xa1, 0x95,
- 0xa9, 0x98, 0x36, 0x77, 0x32, 0xbe, 0x8a, 0xb0, 0x10, 0x1f, 0x3c, 0x85, 0x7e, 0x89, 0x85, 0xe6,
- 0xa0, 0x7e, 0x8c, 0xf3, 0x9b, 0x15, 0x9f, 0x22, 0xbb, 0x27, 0x5e, 0x98, 0xaa, 0x22, 0xeb, 0x39,
- 0xea, 0xf0, 0xbf, 0xda, 0x63, 0xc3, 0xde, 0x06, 0x73, 0x37, 0x0d, 0xc3, 0x42, 0x31, 0x20, 0x2c,
- 0x57, 0x0c, 0x08, 0x9b, 0x14, 0x5a, 0xed, 0xc2, 0x42, 0xfb, 0xc5, 0x80, 0xf9, 0x9d, 0x13, 0x1c,
- 0xf3, 0x97, 0x94, 0x93, 0x21, 0xf1, 0x3d, 0x4e, 0x68, 0x8c, 0xee, 0x83, 0x49, 0xc3, 0xc0, 0xbd,
- 0xb0, 0x52, 0x3b, 0x34, 0xcc, 0xbc, 0xbe, 0x0f, 0x66, 0x8c, 0x4f, 0xdd, 0x0b, 0xcd, 0x75, 0x62,
- 0x7c, 0xaa, 0xa4, 0xd7, 0xa1, 0x1f, 0xe0, 0x10, 0x73, 0xec, 0x16, 0xb7, 0x23, 0xae, 0xae, 0xa7,
- 0x88, 0x5b, 0xea, 0x3a, 0xee, 0xc2, 0x35, 0x01, 0x39, 0xf6, 0x18, 0x8e, 0xb9, 0x3b, 0xf6, 0xf8,
- 0x91, 0xbc, 0x13, 0xd3, 0xe9, 0xc7, 0xf8, 0xf4, 0xb5, 0xa4, 0xbe, 0xf6, 0xf8, 0x91, 0xfd, 0x87,
- 0x01, 0x66, 0x71, 0x99, 0xe8, 0x26, 0xb4, 0x85, 0x59, 0x97, 0x04, 0x59, 0x26, 0x5a, 0xe2, 0xb8,
- 0x17, 0x88, 0xce, 0xa0, 0xc3, 0x61, 0x82, 0xb9, 0x74, 0xaf, 0xee, 0x64, 0x27, 0x51, 0x59, 0x09,
- 0xf9, 0x4e, 0x35, 0x43, 0xc3, 0x91, 0xdf, 0x22, 0xe3, 0x11, 0x27, 0x11, 0x96, 0x06, 0xeb, 0x8e,
- 0x3a, 0xa0, 0x05, 0x68, 0x62, 0x97, 0x7b, 0x23, 0x59, 0xe5, 0xa6, 0xd3, 0xc0, 0x6f, 0xbc, 0x11,
- 0xfa, 0x27, 0x5c, 0x4d, 0x68, 0xca, 0x7c, 0xec, 0xe6, 0x66, 0x5b, 0x92, 0xdb, 0x53, 0xd4, 0x5d,
- 0x65, 0xdc, 0x86, 0xfa, 0x90, 0x04, 0x56, 0x5b, 0x26, 0x66, 0xae, 0x5c, 0x84, 0x7b, 0x81, 0x23,
- 0x98, 0xe8, 0x3f, 0x00, 0x05, 0x52, 0x60, 0x75, 0xce, 0x11, 0x35, 0x73, 0xdc, 0xc0, 0xfe, 0x02,
- 0x5a, 0x19, 0xfc, 0x12, 0x98, 0x27, 0x34, 0x4c, 0xa3, 0x22, 0xec, 0xbe, 0xd3, 0x51, 0x84, 0xbd,
- 0x00, 0xdd, 0x02, 0x39, 0xeb, 0x5c, 0x51, 0x55, 0x35, 0x19, 0xa4, 0xcc, 0xd0, 0x67, 0x58, 0x4e,
- 0x0b, 0x9f, 0xd2, 0x63, 0xa2, 0xa2, 0x6f, 0x3b, 0xd9, 0xc9, 0xfe, 0xbd, 0x06, 0x57, 0xcb, 0xe5,
- 0x2e, 0x4c, 0x48, 0x14, 0x99, 0x2b, 0x43, 0xc2, 0x48, 0xd8, 0x83, 0x52, 0xbe, 0x6a, 0x7a, 0xbe,
- 0x72, 0x95, 0x88, 0x06, 0xca, 0x40, 0x5f, 0xa9, 0xbc, 0xa0, 0x01, 0x16, 0xd5, 0x9a, 0x92, 0x40,
- 0x26, 0xb8, 0xef, 0x88, 0x4f, 0x41, 0x19, 0x91, 0x20, 0x1b, 0x21, 0xe2, 0x53, 0xba, 0xc7, 0x24,
- 0x6e, 0x4b, 0x5d, 0x99, 0x3a, 0x89, 0x2b, 0x8b, 0x04, 0xb5, 0xad, 0xee, 0x41, 0x7c, 0xa3, 0x35,
- 0xe8, 0x32, 0x3c, 0x0e, 0xb3, 0xea, 0x95, 0xe9, 0x33, 0x1d, 0x9d, 0x84, 0x56, 0x01, 0x7c, 0x1a,
- 0x86, 0xd8, 0x97, 0x02, 0xa6, 0x14, 0xd0, 0x28, 0xa2, 0x72, 0x38, 0x0f, 0xdd, 0x04, 0xfb, 0x16,
- 0xac, 0x19, 0x1b, 0x4d, 0xa7, 0xc5, 0x79, 0x78, 0x80, 0x7d, 0x11, 0x47, 0x9a, 0x60, 0xe6, 0xca,
- 0x01, 0xd4, 0x95, 0x7a, 0x1d, 0x41, 0x90, 0xa3, 0x72, 0x05, 0x60, 0xc4, 0x68, 0x3a, 0x56, 0xdc,
- 0xde, 0x5a, 0x5d, 0xcc, 0x63, 0x49, 0x91, 0xec, 0x3b, 0x70, 0x35, 0x79, 0x17, 0x85, 0x24, 0x3e,
- 0x76, 0xb9, 0xc7, 0x46, 0x98, 0x5b, 0x7d, 0x55, 0xc3, 0x19, 0xf5, 0x8d, 0x24, 0xda, 0x5f, 0x02,
- 0xda, 0x62, 0xd8, 0xe3, 0xf8, 0x2f, 0xac, 0x9e, 0x0f, 0xec, 0xee, 0xeb, 0xb0, 0x50, 0x82, 0x56,
- 0x53, 0x58, 0x58, 0x7c, 0x3b, 0x0e, 0x3e, 0x96, 0xc5, 0x12, 0x74, 0x66, 0xf1, 0xbd, 0x01, 0x68,
- 0x5b, 0x36, 0xf8, 0xdf, 0xdb, 0xaf, 0xa2, 0xe5, 0xc4, 0xdc, 0x57, 0x03, 0x24, 0xf0, 0xb8, 0x97,
- 0x6d, 0xa6, 0x1e, 0x49, 0x14, 0xfe, 0xb6, 0xc7, 0xbd, 0x6c, 0x3b, 0x30, 0xec, 0xa7, 0x4c, 0x2c,
- 0x2b, 0x59, 0x57, 0x72, 0x3b, 0x38, 0x39, 0x09, 0x3d, 0x82, 0x1b, 0x64, 0x14, 0x53, 0x86, 0x27,
- 0x62, 0x2e, 0x66, 0x8c, 0x32, 0x59, 0x6f, 0x1d, 0x67, 0x51, 0x71, 0x0b, 0x85, 0x1d, 0xc1, 0x13,
- 0xe1, 0x95, 0xc2, 0xc8, 0xc2, 0xfb, 0xd1, 0x00, 0xeb, 0x19, 0xa7, 0x11, 0xf1, 0x1d, 0x2c, 0xdc,
- 0x2c, 0x05, 0xb9, 0x0e, 0x7d, 0x31, 0x4c, 0xa7, 0x03, 0xed, 0xd1, 0x30, 0x98, 0x2c, 0xab, 0x5b,
- 0x20, 0xe6, 0xa9, 0xab, 0xc5, 0xdb, 0xa6, 0x61, 0x20, 0xcb, 0x68, 0x1d, 0xc4, 0xd0, 0xd3, 0xf4,
- 0xd5, 0xea, 0xee, 0xc5, 0xf8, 0xb4, 0xa4, 0x2f, 0x84, 0xa4, 0xbe, 0x9a, 0x94, 0xed, 0x18, 0x9f,
- 0x0a, 0x7d, 0x7b, 0x09, 0x6e, 0xcd, 0xf0, 0x2d, 0xf3, 0xfc, 0x67, 0x03, 0x16, 0x9e, 0x25, 0x09,
- 0x19, 0xc5, 0x9f, 0xcb, 0x99, 0x91, 0x3b, 0xbd, 0x08, 0x4d, 0x9f, 0xa6, 0x31, 0x97, 0xce, 0x36,
- 0x1d, 0x75, 0x98, 0x6a, 0xa3, 0x5a, 0xa5, 0x8d, 0xa6, 0x1a, 0xb1, 0x5e, 0x6d, 0x44, 0xad, 0xd1,
- 0x1a, 0xa5, 0x46, 0xfb, 0x07, 0x74, 0xc5, 0x75, 0xba, 0x3e, 0x8e, 0x39, 0x66, 0xd9, 0x98, 0x05,
- 0x41, 0xda, 0x92, 0x14, 0xfb, 0x7b, 0x03, 0x16, 0xcb, 0x9e, 0x66, 0x6f, 0x8a, 0x73, 0xa7, 0xbe,
- 0x18, 0x33, 0x2c, 0xcc, 0xdc, 0x14, 0x9f, 0xa2, 0x61, 0xc7, 0xe9, 0x61, 0x48, 0x7c, 0x57, 0x30,
- 0x94, 0x7b, 0xa6, 0xa2, 0xbc, 0x65, 0xe1, 0x24, 0xe8, 0x86, 0x1e, 0x34, 0x82, 0x86, 0x97, 0xf2,
- 0xa3, 0x7c, 0xf2, 0x8b, 0x6f, 0xfb, 0x11, 0x2c, 0xa8, 0x67, 0x5e, 0x39, 0x6b, 0x2b, 0x00, 0xc5,
- 0x2c, 0x56, 0x2f, 0x1c, 0xd3, 0x31, 0xf3, 0x61, 0x9c, 0xd8, 0xff, 0x07, 0x73, 0x9f, 0xaa, 0x44,
- 0x24, 0xe8, 0x01, 0x98, 0x61, 0x7e, 0xc8, 0x1e, 0x43, 0x68, 0xd2, 0x54, 0xb9, 0x9c, 0x33, 0x11,
- 0xb2, 0x9f, 0x42, 0x27, 0x27, 0xe7, 0xb1, 0x19, 0xe7, 0xc5, 0x56, 0x9b, 0x8a, 0xcd, 0xfe, 0xcd,
- 0x80, 0xc5, 0xb2, 0xcb, 0x59, 0xfa, 0xde, 0x42, 0xbf, 0x30, 0xe1, 0x46, 0xde, 0x38, 0xf3, 0xe5,
- 0x81, 0xee, 0x4b, 0x55, 0xad, 0x70, 0x30, 0x79, 0xe1, 0x8d, 0x55, 0x49, 0xf5, 0x42, 0x8d, 0x34,
- 0x78, 0x03, 0xf3, 0x15, 0x91, 0x19, 0xef, 0x9b, 0x7b, 0xfa, 0xfb, 0xa6, 0xf4, 0x46, 0x2b, 0xb4,
- 0xf5, 0x47, 0xcf, 0x13, 0xb8, 0xa9, 0xfa, 0x6f, 0xab, 0x28, 0xba, 0x3c, 0xf7, 0xe5, 0xda, 0x34,
- 0xa6, 0x6b, 0xd3, 0x1e, 0x80, 0x55, 0x55, 0xcd, 0xba, 0x60, 0x04, 0xf3, 0x07, 0xdc, 0xe3, 0x24,
- 0xe1, 0xc4, 0x2f, 0x1e, 0xdb, 0x53, 0xc5, 0x6c, 0x5c, 0xb6, 0x55, 0xaa, 0xed, 0x30, 0x07, 0x75,
- 0xce, 0xf3, 0x3a, 0x13, 0x9f, 0xe2, 0x16, 0x90, 0x6e, 0x29, 0xbb, 0x83, 0x8f, 0x60, 0x4a, 0xd4,
- 0x03, 0xa7, 0xdc, 0x0b, 0xd5, 0xd6, 0x6e, 0xc8, 0xad, 0x6d, 0x4a, 0x8a, 0x5c, 0xdb, 0x6a, 0xb1,
- 0x05, 0x8a, 0xdb, 0x54, 0x3b, 0x5d, 0x10, 0x24, 0x73, 0x05, 0x40, 0xb6, 0x94, 0xea, 0x86, 0x96,
- 0xd2, 0x15, 0x94, 0x2d, 0x41, 0xb0, 0x57, 0x61, 0xf9, 0x53, 0xcc, 0xc5, 0xfb, 0x83, 0x6d, 0xd1,
- 0x78, 0x48, 0x46, 0x29, 0xf3, 0xb4, 0xab, 0xb0, 0x7f, 0x30, 0x60, 0xe5, 0x1c, 0x81, 0x2c, 0x60,
- 0x0b, 0xda, 0x91, 0x97, 0x70, 0xcc, 0xf2, 0x2e, 0xc9, 0x8f, 0xd3, 0xa9, 0xa8, 0x5d, 0x96, 0x8a,
- 0x7a, 0x25, 0x15, 0xd7, 0xa1, 0x15, 0x79, 0x67, 0x6e, 0x74, 0x98, 0x3d, 0x30, 0x9a, 0x91, 0x77,
- 0xf6, 0xe2, 0xf0, 0xe1, 0xfb, 0x36, 0xf4, 0x0e, 0xb0, 0x77, 0x8a, 0x71, 0x20, 0x1d, 0x43, 0xa3,
- 0xbc, 0x21, 0xca, 0x3f, 0xd5, 0xd0, 0x9d, 0xe9, 0xca, 0x9f, 0xf9, 0xdb, 0x70, 0x70, 0xf7, 0x32,
- 0xb1, 0xac, 0xb6, 0xae, 0xa0, 0x7d, 0xe8, 0x6a, 0xbf, 0x85, 0xd0, 0xb2, 0xa6, 0x58, 0xf9, 0x89,
- 0x37, 0x58, 0x39, 0x87, 0xab, 0xa3, 0x69, 0x3b, 0x5d, 0x47, 0xab, 0xbe, 0x22, 0x74, 0xb4, 0x59,
- 0x0f, 0x01, 0x89, 0xa6, 0xed, 0x6b, 0x1d, 0xad, 0xfa, 0x42, 0xd0, 0xd1, 0x66, 0x2d, 0x79, 0x89,
- 0xa6, 0xad, 0x47, 0x1d, 0xad, 0xba, 0xfc, 0x75, 0xb4, 0x59, 0x3b, 0xf5, 0x0a, 0xfa, 0x1a, 0xe6,
- 0x2b, 0x8b, 0x0b, 0xd9, 0x13, 0xad, 0xf3, 0x36, 0xee, 0x60, 0xfd, 0x42, 0x99, 0x02, 0xff, 0x15,
- 0xf4, 0xf4, 0x85, 0x82, 0x34, 0x87, 0x66, 0xac, 0xc4, 0xc1, 0xea, 0x79, 0x6c, 0x1d, 0x50, 0x9f,
- 0x95, 0x3a, 0xe0, 0x8c, 0x6d, 0xa1, 0x03, 0xce, 0x1a, 0xb1, 0xf6, 0x15, 0xf4, 0x15, 0xcc, 0x4d,
- 0xcf, 0x2c, 0x74, 0x7b, 0x3a, 0x6d, 0x95, 0x51, 0x38, 0xb0, 0x2f, 0x12, 0x29, 0xc0, 0xf7, 0x00,
- 0x26, 0xa3, 0x08, 0x2d, 0x4d, 0x74, 0x2a, 0xa3, 0x70, 0xb0, 0x3c, 0x9b, 0x59, 0x40, 0x7d, 0x03,
- 0xd7, 0x67, 0xf6, 0x3b, 0xd2, 0x9a, 0xe4, 0xa2, 0x89, 0x31, 0xf8, 0xd7, 0xa5, 0x72, 0xb9, 0xad,
- 0xe7, 0xab, 0x30, 0x97, 0xa8, 0x36, 0x1e, 0x26, 0x9b, 0x7e, 0x48, 0x70, 0xcc, 0x9f, 0x83, 0xd4,
- 0x78, 0xcd, 0x28, 0xa7, 0x87, 0x2d, 0xf9, 0x1f, 0xcf, 0x7f, 0xff, 0x0c, 0x00, 0x00, 0xff, 0xff,
- 0x0e, 0xa9, 0xb5, 0x68, 0xf2, 0x11, 0x00, 0x00,
+ // 1603 bytes of a gzipped FileDescriptorProto
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0xcd, 0x6f, 0xdc, 0x44,
+ 0x14, 0x8f, 0xf7, 0xdb, 0x6f, 0x77, 0xdb, 0x64, 0x92, 0xb6, 0xdb, 0xcd, 0x07, 0xa9, 0x43, 0x4b,
+ 0x10, 0x55, 0xa8, 0x42, 0x0f, 0x2d, 0x85, 0x43, 0x9b, 0x0f, 0x14, 0x91, 0x7e, 0xc8, 0x69, 0x11,
+ 0x08, 0x09, 0xcb, 0xb1, 0x67, 0x37, 0x43, 0x6c, 0xcf, 0x32, 0x1e, 0x27, 0x29, 0x7f, 0x02, 0x47,
+ 0x8e, 0x48, 0x9c, 0xf9, 0x27, 0x10, 0x17, 0x84, 0xf8, 0x6f, 0x38, 0x72, 0x46, 0x33, 0x63, 0x7b,
+ 0xc7, 0xeb, 0x4d, 0xd2, 0x0a, 0xf5, 0xe6, 0x79, 0xdf, 0xef, 0xcd, 0x7b, 0xbf, 0x37, 0xbb, 0xd0,
+ 0x1e, 0x90, 0x00, 0xb3, 0x8d, 0x11, 0xa3, 0x9c, 0xa2, 0x96, 0x3c, 0x38, 0xa3, 0x43, 0xeb, 0x39,
+ 0x2c, 0xee, 0x53, 0x7a, 0x9c, 0x8c, 0xb6, 0x09, 0xc3, 0x1e, 0xa7, 0xec, 0xf5, 0x4e, 0xc4, 0xd9,
+ 0x6b, 0x1b, 0xff, 0x90, 0xe0, 0x98, 0xa3, 0x25, 0x30, 0xfd, 0x8c, 0xd1, 0x33, 0x56, 0x8d, 0x75,
+ 0xd3, 0x1e, 0x13, 0x10, 0x82, 0x5a, 0xe4, 0x86, 0xb8, 0x57, 0x91, 0x0c, 0xf9, 0x6d, 0xed, 0xc0,
+ 0xd2, 0x74, 0x83, 0xf1, 0x88, 0x46, 0x31, 0x46, 0xb7, 0xa1, 0x8e, 0x05, 0x41, 0x5a, 0x6b, 0x6f,
+ 0x5e, 0xdd, 0xc8, 0x42, 0xd9, 0x50, 0x72, 0x8a, 0x6b, 0xfd, 0x61, 0x00, 0xda, 0x27, 0x31, 0x17,
+ 0x44, 0x82, 0xe3, 0x37, 0x8b, 0xe7, 0x3a, 0x34, 0x46, 0x0c, 0x0f, 0xc8, 0x59, 0x1a, 0x51, 0x7a,
+ 0x42, 0x77, 0x61, 0x2e, 0xe6, 0x2e, 0xe3, 0xbb, 0x8c, 0x86, 0xbb, 0x24, 0xc0, 0xcf, 0x44, 0xd0,
+ 0x55, 0x29, 0x52, 0x66, 0xa0, 0x0d, 0x40, 0x24, 0xf2, 0x82, 0x24, 0x26, 0x27, 0xf8, 0x20, 0xe3,
+ 0xf6, 0x6a, 0xab, 0xc6, 0x7a, 0xcb, 0x9e, 0xc2, 0x41, 0x0b, 0x50, 0x0f, 0x48, 0x48, 0x78, 0xaf,
+ 0xbe, 0x6a, 0xac, 0x77, 0x6d, 0x75, 0xb0, 0x3e, 0x83, 0xf9, 0x42, 0xfc, 0x6f, 0x97, 0xfe, 0xaf,
+ 0x15, 0xa8, 0x4b, 0x42, 0x5e, 0x63, 0x63, 0x5c, 0x63, 0x74, 0x0b, 0x3a, 0x24, 0x76, 0xc6, 0x85,
+ 0xa8, 0xc8, 0xd8, 0xda, 0x24, 0xce, 0x6b, 0x8e, 0x3e, 0x82, 0x86, 0x77, 0x94, 0x44, 0xc7, 0x71,
+ 0xaf, 0xba, 0x5a, 0x5d, 0x6f, 0x6f, 0xce, 0x8f, 0x1d, 0x89, 0x44, 0xb7, 0x04, 0xcf, 0x4e, 0x45,
+ 0xd0, 0x03, 0x00, 0x97, 0x73, 0x46, 0x0e, 0x13, 0x8e, 0x63, 0x99, 0x69, 0x7b, 0xb3, 0xa7, 0x29,
+ 0x24, 0x31, 0x7e, 0x9c, 0xf3, 0x6d, 0x4d, 0x16, 0x3d, 0x84, 0x16, 0x3e, 0xe3, 0x38, 0xf2, 0xb1,
+ 0xdf, 0xab, 0x4b, 0x47, 0xcb, 0x13, 0x19, 0x6d, 0xec, 0xa4, 0x7c, 0x95, 0x5f, 0x2e, 0xde, 0x7f,
+ 0x04, 0xdd, 0x02, 0x0b, 0xcd, 0x42, 0xf5, 0x18, 0x67, 0xb7, 0x2a, 0x3e, 0x45, 0x65, 0x4f, 0xdc,
+ 0x20, 0x51, 0x0d, 0xd6, 0xb1, 0xd5, 0xe1, 0xd3, 0xca, 0x03, 0xc3, 0xda, 0x06, 0x73, 0x37, 0x09,
+ 0x82, 0x5c, 0xd1, 0x27, 0x2c, 0x53, 0xf4, 0x09, 0x1b, 0x57, 0xb9, 0x72, 0x61, 0x95, 0x7f, 0x37,
+ 0x60, 0x6e, 0xe7, 0x04, 0x47, 0xfc, 0x19, 0xe5, 0x64, 0x40, 0x3c, 0x97, 0x13, 0x1a, 0xa1, 0xbb,
+ 0x60, 0xd2, 0xc0, 0x77, 0x2e, 0xbc, 0xa6, 0x16, 0x0d, 0xd2, 0xa8, 0xef, 0x82, 0x19, 0xe1, 0x53,
+ 0xe7, 0x42, 0x77, 0xad, 0x08, 0x9f, 0x2a, 0xe9, 0x35, 0xe8, 0xfa, 0x38, 0xc0, 0x1c, 0x3b, 0xf9,
+ 0xed, 0x88, 0xab, 0xeb, 0x28, 0xe2, 0x96, 0xba, 0x8e, 0x3b, 0x70, 0x55, 0x98, 0x1c, 0xb9, 0x0c,
+ 0x47, 0xdc, 0x19, 0xb9, 0xfc, 0x48, 0xde, 0x89, 0x69, 0x77, 0x23, 0x7c, 0xfa, 0x42, 0x52, 0x5f,
+ 0xb8, 0xfc, 0xc8, 0xfa, 0xd7, 0x00, 0x33, 0xbf, 0x4c, 0x74, 0x03, 0x9a, 0xc2, 0xad, 0x43, 0xfc,
+ 0xb4, 0x12, 0x0d, 0x71, 0xdc, 0xf3, 0xc5, 0x54, 0xd0, 0xc1, 0x20, 0xc6, 0x5c, 0x86, 0x57, 0xb5,
+ 0xd3, 0x93, 0xe8, 0xac, 0x98, 0xfc, 0xa8, 0x06, 0xa1, 0x66, 0xcb, 0x6f, 0x51, 0xf1, 0x90, 0x93,
+ 0x10, 0x4b, 0x87, 0x55, 0x5b, 0x1d, 0xd0, 0x3c, 0xd4, 0xb1, 0xc3, 0xdd, 0xa1, 0xec, 0x70, 0xd3,
+ 0xae, 0xe1, 0x97, 0xee, 0x10, 0xbd, 0x0f, 0x57, 0x62, 0x9a, 0x30, 0x0f, 0x3b, 0x99, 0xdb, 0x86,
+ 0xe4, 0x76, 0x14, 0x75, 0x57, 0x39, 0xb7, 0xa0, 0x3a, 0x20, 0x7e, 0xaf, 0x29, 0x0b, 0x33, 0x5b,
+ 0x6c, 0xc2, 0x3d, 0xdf, 0x16, 0x4c, 0xf4, 0x31, 0x40, 0x6e, 0xc9, 0xef, 0xb5, 0xce, 0x11, 0x35,
+ 0x33, 0xbb, 0xbe, 0xf5, 0x35, 0x34, 0x52, 0xf3, 0x8b, 0x60, 0x9e, 0xd0, 0x20, 0x09, 0xf3, 0xb4,
+ 0xbb, 0x76, 0x4b, 0x11, 0xf6, 0x7c, 0x74, 0x13, 0x24, 0xce, 0x39, 0xa2, 0xab, 0x2a, 0x32, 0x49,
+ 0x59, 0xa1, 0x2f, 0xb1, 0x44, 0x0a, 0x8f, 0xd2, 0x63, 0xa2, 0xb2, 0x6f, 0xda, 0xe9, 0xc9, 0xfa,
+ 0xa7, 0x02, 0x57, 0x8a, 0xed, 0x2e, 0x5c, 0x48, 0x2b, 0xb2, 0x56, 0x86, 0x34, 0x23, 0xcd, 0x1e,
+ 0x14, 0xea, 0x55, 0xd1, 0xeb, 0x95, 0xa9, 0x84, 0xd4, 0x57, 0x0e, 0xba, 0x4a, 0xe5, 0x29, 0xf5,
+ 0xb1, 0xe8, 0xd6, 0x84, 0xf8, 0xb2, 0xc0, 0x5d, 0x5b, 0x7c, 0x0a, 0xca, 0x90, 0xf8, 0x29, 0x7c,
+ 0x88, 0x4f, 0x19, 0x1e, 0x93, 0x76, 0x1b, 0xea, 0xca, 0xd4, 0x49, 0x5c, 0x59, 0x28, 0xa8, 0x4d,
+ 0x75, 0x0f, 0xe2, 0x1b, 0xad, 0x42, 0x9b, 0xe1, 0x51, 0x90, 0x76, 0xaf, 0x2c, 0x9f, 0x69, 0xeb,
+ 0x24, 0xb4, 0x02, 0xe0, 0xd1, 0x20, 0xc0, 0x9e, 0x14, 0x30, 0xa5, 0x80, 0x46, 0x11, 0x9d, 0xc3,
+ 0x79, 0xe0, 0xc4, 0xd8, 0xeb, 0xc1, 0xaa, 0xb1, 0x5e, 0xb7, 0x1b, 0x9c, 0x07, 0x07, 0xd8, 0x13,
+ 0x79, 0x24, 0x31, 0x66, 0x8e, 0x04, 0xa0, 0xb6, 0xd4, 0x6b, 0x09, 0x82, 0x84, 0xc9, 0x65, 0x80,
+ 0x21, 0xa3, 0xc9, 0x48, 0x71, 0x3b, 0xab, 0x55, 0x81, 0xc5, 0x92, 0x22, 0xd9, 0xb7, 0xe1, 0x4a,
+ 0xfc, 0x3a, 0x0c, 0x48, 0x74, 0xec, 0x70, 0x97, 0x0d, 0x31, 0xef, 0x75, 0x55, 0x0f, 0xa7, 0xd4,
+ 0x97, 0x92, 0x68, 0x7d, 0x03, 0x68, 0x8b, 0x61, 0x97, 0xe3, 0xb7, 0x58, 0x3b, 0x6f, 0x38, 0xdd,
+ 0xd7, 0x60, 0xbe, 0x60, 0x5a, 0x21, 0xb0, 0xf0, 0xf8, 0x6a, 0xe4, 0xbf, 0x2b, 0x8f, 0x05, 0xd3,
+ 0xa9, 0xc7, 0xbf, 0x0c, 0x40, 0xdb, 0x72, 0xc0, 0xff, 0xdf, 0x6e, 0x15, 0x23, 0x27, 0x70, 0x5f,
+ 0x01, 0x88, 0xef, 0x72, 0x37, 0xdd, 0x4a, 0x1d, 0x12, 0x2b, 0xfb, 0xdb, 0x2e, 0x77, 0xd3, 0xed,
+ 0xc0, 0xb0, 0x97, 0x30, 0xb1, 0xa8, 0x64, 0x5f, 0xc9, 0xed, 0x60, 0x67, 0x24, 0x74, 0x1f, 0xae,
+ 0x93, 0x61, 0x44, 0x19, 0x1e, 0x8b, 0x39, 0x98, 0x31, 0xca, 0x64, 0xbf, 0xb5, 0xec, 0x05, 0xc5,
+ 0xcd, 0x15, 0x76, 0x04, 0x4f, 0xa4, 0x57, 0x48, 0x23, 0x4d, 0xef, 0x17, 0x03, 0x7a, 0x8f, 0x39,
+ 0x0d, 0x89, 0x67, 0x63, 0x11, 0x66, 0x21, 0xc9, 0x35, 0xe8, 0x0a, 0x30, 0x9d, 0x4c, 0xb4, 0x43,
+ 0x03, 0x7f, 0xbc, 0xac, 0x6e, 0x82, 0xc0, 0x53, 0x47, 0xcb, 0xb7, 0x49, 0x03, 0x5f, 0xb6, 0xd1,
+ 0x1a, 0x08, 0xd0, 0xd3, 0xf4, 0xd5, 0xda, 0xee, 0x44, 0xf8, 0xb4, 0xa0, 0x2f, 0x84, 0xa4, 0xbe,
+ 0x42, 0xca, 0x66, 0x84, 0x4f, 0x85, 0xbe, 0xb5, 0x08, 0x37, 0xa7, 0xc4, 0x96, 0x46, 0xfe, 0x9b,
+ 0x01, 0xf3, 0x8f, 0xe3, 0x98, 0x0c, 0xa3, 0xaf, 0x24, 0x66, 0x64, 0x41, 0x2f, 0x40, 0xdd, 0xa3,
+ 0x49, 0xc4, 0x65, 0xb0, 0x75, 0x5b, 0x1d, 0x26, 0xc6, 0xa8, 0x52, 0x1a, 0xa3, 0x89, 0x41, 0xac,
+ 0x96, 0x07, 0x51, 0x1b, 0xb4, 0x5a, 0x61, 0xd0, 0xde, 0x83, 0xb6, 0xb8, 0x4e, 0xc7, 0xc3, 0x11,
+ 0xc7, 0x2c, 0x85, 0x59, 0x10, 0xa4, 0x2d, 0x49, 0xb1, 0x7e, 0x32, 0x60, 0xa1, 0x18, 0x69, 0xfa,
+ 0x9e, 0x38, 0x17, 0xf5, 0x05, 0xcc, 0xb0, 0x20, 0x0d, 0x53, 0x7c, 0x8a, 0x81, 0x1d, 0x25, 0x87,
+ 0x01, 0xf1, 0x1c, 0xc1, 0x50, 0xe1, 0x99, 0x8a, 0xf2, 0x8a, 0x05, 0xe3, 0xa4, 0x6b, 0x7a, 0xd2,
+ 0x08, 0x6a, 0x6e, 0xc2, 0x8f, 0x32, 0xe4, 0x17, 0xdf, 0xd6, 0x7d, 0x98, 0x57, 0x4f, 0xbc, 0x62,
+ 0xd5, 0x96, 0x01, 0x72, 0x2c, 0x8e, 0x7b, 0x86, 0x02, 0x84, 0x0c, 0x8c, 0x63, 0xeb, 0x73, 0x30,
+ 0xf7, 0xa9, 0x2a, 0x44, 0x8c, 0xee, 0x81, 0x19, 0x64, 0x07, 0x29, 0xda, 0xde, 0x44, 0xe3, 0xa1,
+ 0xca, 0xe4, 0xec, 0xb1, 0x90, 0xf5, 0x08, 0x5a, 0x19, 0x39, 0xcb, 0xcd, 0x38, 0x2f, 0xb7, 0xca,
+ 0x44, 0x6e, 0xd6, 0x9f, 0x06, 0x2c, 0x14, 0x43, 0x4e, 0xcb, 0xf7, 0x0a, 0xba, 0xb9, 0x0b, 0x27,
+ 0x74, 0x47, 0x69, 0x2c, 0xf7, 0xf4, 0x58, 0xca, 0x6a, 0x79, 0x80, 0xf1, 0x53, 0x77, 0xa4, 0x5a,
+ 0xaa, 0x13, 0x68, 0xa4, 0xfe, 0x4b, 0x98, 0x2b, 0x89, 0x4c, 0x79, 0xdf, 0x7c, 0xa8, 0xbf, 0x6f,
+ 0x0a, 0x6f, 0xb4, 0x5c, 0x5b, 0x7f, 0xf4, 0x3c, 0x84, 0x1b, 0x6a, 0xfe, 0xb6, 0xf2, 0xa6, 0xcb,
+ 0x6a, 0x5f, 0xec, 0x4d, 0x63, 0xb2, 0x37, 0xad, 0x3e, 0xf4, 0xca, 0xaa, 0xe9, 0x14, 0x0c, 0x61,
+ 0xee, 0x80, 0xbb, 0x9c, 0xc4, 0x9c, 0x78, 0xf9, 0x43, 0x7b, 0xa2, 0x99, 0x8d, 0xcb, 0xb6, 0x4a,
+ 0x79, 0x1c, 0x66, 0xa1, 0xca, 0x79, 0xd6, 0x67, 0xe2, 0x53, 0xdc, 0x02, 0xd2, 0x3d, 0xa5, 0x77,
+ 0xf0, 0x0e, 0x5c, 0x89, 0x7e, 0xe0, 0x94, 0xbb, 0x81, 0xda, 0xda, 0x35, 0xb9, 0xb5, 0x4d, 0x49,
+ 0x91, 0x6b, 0x5b, 0x2d, 0x36, 0x5f, 0x71, 0xeb, 0x6a, 0xa7, 0x0b, 0x82, 0x64, 0x2e, 0x03, 0xc8,
+ 0x91, 0x52, 0xd3, 0xd0, 0x50, 0xba, 0x82, 0xb2, 0x25, 0x08, 0xd6, 0x0a, 0x2c, 0x7d, 0x81, 0xb9,
+ 0x78, 0x7f, 0xb0, 0x2d, 0x1a, 0x0d, 0xc8, 0x30, 0x61, 0xae, 0x76, 0x15, 0xd6, 0xcf, 0x06, 0x2c,
+ 0x9f, 0x23, 0x90, 0x26, 0xdc, 0x83, 0x66, 0xe8, 0xc6, 0x1c, 0xb3, 0x6c, 0x4a, 0xb2, 0xe3, 0x64,
+ 0x29, 0x2a, 0x97, 0x95, 0xa2, 0x5a, 0x2a, 0xc5, 0x35, 0x68, 0x84, 0xee, 0x99, 0x13, 0x1e, 0xa6,
+ 0x0f, 0x8c, 0x7a, 0xe8, 0x9e, 0x3d, 0x3d, 0xdc, 0xfc, 0xbb, 0x09, 0x9d, 0x03, 0xec, 0x9e, 0x62,
+ 0xec, 0xcb, 0xc0, 0xd0, 0x30, 0x1b, 0x88, 0xe2, 0xcf, 0x34, 0x74, 0x7b, 0xb2, 0xf3, 0xa7, 0xfe,
+ 0x2e, 0xec, 0xdf, 0xb9, 0x4c, 0x2c, 0xed, 0xad, 0x19, 0xf4, 0x0c, 0xda, 0xda, 0xef, 0x20, 0xb4,
+ 0xa4, 0x29, 0x96, 0x7e, 0xde, 0xf5, 0x97, 0xcf, 0xe1, 0x66, 0xd6, 0xee, 0x19, 0x68, 0x1f, 0xda,
+ 0xda, 0x56, 0xd7, 0xed, 0x95, 0xdf, 0x11, 0xba, 0xbd, 0x69, 0x4f, 0x81, 0x19, 0x61, 0x4d, 0xdb,
+ 0xd8, 0xba, 0xb5, 0xf2, 0x1b, 0x41, 0xb7, 0x36, 0x6d, 0xcd, 0x4b, 0x6b, 0xda, 0x82, 0xd4, 0xad,
+ 0x95, 0xd7, 0xbf, 0x6e, 0x6d, 0xda, 0x56, 0x9d, 0x41, 0xdf, 0xc1, 0x5c, 0x69, 0x75, 0x21, 0x6b,
+ 0xac, 0x75, 0xde, 0xce, 0xed, 0xaf, 0x5d, 0x28, 0x93, 0xdb, 0x7f, 0x0e, 0x1d, 0x7d, 0xa5, 0x20,
+ 0x2d, 0xa0, 0x29, 0x4b, 0xb1, 0xbf, 0x72, 0x1e, 0x5b, 0x37, 0xa8, 0xa3, 0xa5, 0x6e, 0x70, 0xca,
+ 0xbe, 0xd0, 0x0d, 0x4e, 0x03, 0x59, 0x6b, 0x06, 0x7d, 0x0b, 0xb3, 0x93, 0xa8, 0x85, 0x6e, 0x4d,
+ 0x96, 0xad, 0x04, 0x86, 0x7d, 0xeb, 0x22, 0x91, 0xdc, 0xf8, 0x1e, 0xc0, 0x18, 0x8c, 0xd0, 0xe2,
+ 0x58, 0xa7, 0x04, 0x86, 0xfd, 0xa5, 0xe9, 0xcc, 0xdc, 0xd4, 0xf7, 0x70, 0x6d, 0xea, 0xc4, 0x23,
+ 0x6d, 0x4c, 0x2e, 0xc2, 0x8c, 0xfe, 0x07, 0x97, 0xca, 0x65, 0xbe, 0x9e, 0xac, 0xc0, 0x6c, 0xac,
+ 0x06, 0x79, 0x10, 0x6f, 0x78, 0x01, 0xc1, 0x11, 0x7f, 0x02, 0x52, 0xe3, 0x05, 0xa3, 0x9c, 0x1e,
+ 0x36, 0xe4, 0x3f, 0x3c, 0x9f, 0xfc, 0x17, 0x00, 0x00, 0xff, 0xff, 0x14, 0x43, 0x9d, 0xb9, 0xf0,
+ 0x11, 0x00, 0x00,
}
diff --git a/weed/s3api/filer_util.go b/weed/s3api/filer_util.go
index b93b603e2..ed9612d35 100644
--- a/weed/s3api/filer_util.go
+++ b/weed/s3api/filer_util.go
@@ -3,6 +3,7 @@ package s3api
import (
"context"
"fmt"
+ "io"
"os"
"strings"
"time"
@@ -89,13 +90,25 @@ func (s3a *S3ApiServer) list(ctx context.Context, parentDirectoryPath, prefix, s
}
glog.V(4).Infof("read directory: %v", request)
- resp, err := client.ListEntries(ctx, request)
+ stream, err := client.ListEntries(ctx, request)
if err != nil {
glog.V(0).Infof("read directory %v: %v", request, err)
return fmt.Errorf("list dir %v: %v", parentDirectoryPath, err)
}
- entries = resp.Entries
+ for {
+ resp, recvErr := stream.Recv()
+ if recvErr != nil {
+ if recvErr == io.EOF {
+ break
+ } else {
+ return recvErr
+ }
+ }
+
+ entries = append(entries, resp.Entry)
+
+ }
return nil
})
diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go
index 1fc8b6b37..aa6849cbd 100644
--- a/weed/s3api/s3api_objects_list_handlers.go
+++ b/weed/s3api/s3api_objects_list_handlers.go
@@ -3,6 +3,7 @@ package s3api
import (
"context"
"fmt"
+ "io"
"net/http"
"net/url"
"path/filepath"
@@ -107,7 +108,7 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, bucket, originalPr
InclusiveStartFrom: false,
}
- resp, err := client.ListEntries(ctx, request)
+ stream, err := client.ListEntries(ctx, request)
if err != nil {
return fmt.Errorf("list buckets: %v", err)
}
@@ -117,7 +118,18 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, bucket, originalPr
var counter int
var lastEntryName string
var isTruncated bool
- for _, entry := range resp.Entries {
+
+ for {
+ resp, recvErr := stream.Recv()
+ if recvErr != nil {
+ if recvErr == io.EOF {
+ break
+ } else {
+ return recvErr
+ }
+ }
+
+ entry := resp.Entry
counter++
if counter > maxKeys {
isTruncated = true
@@ -143,6 +155,7 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, bucket, originalPr
StorageClass: "STANDARD",
})
}
+
}
response = ListBucketResult{
diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go
index b2dd841e2..acfd2260d 100644
--- a/weed/server/filer_grpc_server.go
+++ b/weed/server/filer_grpc_server.go
@@ -33,7 +33,7 @@ func (fs *FilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.L
}, nil
}
-func (fs *FilerServer) ListEntries(ctx context.Context, req *filer_pb.ListEntriesRequest) (*filer_pb.ListEntriesResponse, error) {
+func (fs *FilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream filer_pb.SeaweedFiler_ListEntriesServer) error {
limit := int(req.Limit)
if limit == 0 {
@@ -45,16 +45,15 @@ func (fs *FilerServer) ListEntries(ctx context.Context, req *filer_pb.ListEntrie
paginationLimit = limit
}
- resp := &filer_pb.ListEntriesResponse{}
lastFileName := req.StartFromFileName
includeLastFile := req.InclusiveStartFrom
for limit > 0 {
- entries, err := fs.filer.ListDirectoryEntries(ctx, filer2.FullPath(req.Directory), lastFileName, includeLastFile, paginationLimit)
+ entries, err := fs.filer.ListDirectoryEntries(stream.Context(), filer2.FullPath(req.Directory), lastFileName, includeLastFile, paginationLimit)
if err != nil {
- return nil, err
+ return err
}
if len(entries) == 0 {
- return resp, nil
+ return nil
}
includeLastFile = false
@@ -69,15 +68,19 @@ func (fs *FilerServer) ListEntries(ctx context.Context, req *filer_pb.ListEntrie
}
}
- resp.Entries = append(resp.Entries, &filer_pb.Entry{
- Name: entry.Name(),
- IsDirectory: entry.IsDirectory(),
- Chunks: entry.Chunks,
- Attributes: filer2.EntryAttributeToPb(entry),
- })
+ if err := stream.Send(&filer_pb.ListEntriesResponse{
+ Entry: &filer_pb.Entry{
+ Name: entry.Name(),
+ IsDirectory: entry.IsDirectory(),
+ Chunks: entry.Chunks,
+ Attributes: filer2.EntryAttributeToPb(entry),
+ },
+ }); err != nil {
+ return err
+ }
limit--
if limit == 0 {
- return resp, nil
+ return nil
}
}
@@ -87,7 +90,7 @@ func (fs *FilerServer) ListEntries(ctx context.Context, req *filer_pb.ListEntrie
}
- return resp, nil
+ return nil
}
func (fs *FilerServer) LookupVolume(ctx context.Context, req *filer_pb.LookupVolumeRequest) (*filer_pb.LookupVolumeResponse, error) {
diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go
index b367d78b3..a1fe4bc1c 100644
--- a/weed/server/webdav_server.go
+++ b/weed/server/webdav_server.go
@@ -512,7 +512,7 @@ func (f *WebDavFile) Readdir(count int) (ret []os.FileInfo, err error) {
dir = dir[:len(dir)-1]
}
- err = filer2.ReadDirAllEntries(ctx, f.fs, dir, func(entry *filer_pb.Entry) {
+ err = filer2.ReadDirAllEntries(ctx, f.fs, dir, "", func(entry *filer_pb.Entry, isLast bool) {
fi := FileInfo{
size: int64(filer2.TotalSize(entry.GetChunks())),
name: entry.Name,
diff --git a/weed/shell/command_fs_du.go b/weed/shell/command_fs_du.go
index 5e634c82a..b6e767f17 100644
--- a/weed/shell/command_fs_du.go
+++ b/weed/shell/command_fs_du.go
@@ -3,11 +3,13 @@ package shell
import (
"context"
"fmt"
+ "io"
+
+ "google.golang.org/grpc"
+
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
- "google.golang.org/grpc"
- "io"
)
func init() {
@@ -43,67 +45,42 @@ func (c *commandFsDu) Do(args []string, commandEnv *CommandEnv, writer io.Writer
path = path + "/"
}
- dir, name := filer2.FullPath(path).DirAndName()
-
- return commandEnv.withFilerClient(ctx, filerServer, filerPort, func(client filer_pb.SeaweedFilerClient) error {
- _, _, err = paginateDirectory(ctx, writer, client, dir, name, 1000)
+ var blockCount, byteCount uint64
+ dir, name := filer2.FullPath(path).DirAndName()
+ blockCount, byteCount, err = duTraverseDirectory(ctx, writer, commandEnv.getFilerClient(filerServer, filerPort), dir, name)
- return err
+ if name == "" && err == nil {
+ fmt.Fprintf(writer, "block:%4d\tbyte:%10d\t%s\n", blockCount, byteCount, dir)
+ }
- })
+ return
}
-func paginateDirectory(ctx context.Context, writer io.Writer, client filer_pb.SeaweedFilerClient, dir, name string, paginateSize int) (blockCount uint64, byteCount uint64, err error) {
-
- paginatedCount := -1
- startFromFileName := ""
-
- for paginatedCount == -1 || paginatedCount == paginateSize {
- resp, listErr := client.ListEntries(ctx, &filer_pb.ListEntriesRequest{
- Directory: dir,
- Prefix: name,
- StartFromFileName: startFromFileName,
- InclusiveStartFrom: false,
- Limit: uint32(paginateSize),
- })
- if listErr != nil {
- err = listErr
- return
- }
+func duTraverseDirectory(ctx context.Context, writer io.Writer, filerClient filer2.FilerClient, dir, name string) (blockCount uint64, byteCount uint64, err error) {
- paginatedCount = len(resp.Entries)
-
- for _, entry := range resp.Entries {
- if entry.IsDirectory {
- subDir := fmt.Sprintf("%s/%s", dir, entry.Name)
- if dir == "/" {
- subDir = "/" + entry.Name
- }
- numBlock, numByte, err := paginateDirectory(ctx, writer, client, subDir, "", paginateSize)
- if err == nil {
- blockCount += numBlock
- byteCount += numByte
- }
- } else {
- blockCount += uint64(len(entry.Chunks))
- byteCount += filer2.TotalSize(entry.Chunks)
+ err = filer2.ReadDirAllEntries(ctx, filerClient, dir, name, func(entry *filer_pb.Entry, isLast bool) {
+ if entry.IsDirectory {
+ subDir := fmt.Sprintf("%s/%s", dir, entry.Name)
+ if dir == "/" {
+ subDir = "/" + entry.Name
}
- startFromFileName = entry.Name
-
- if name != "" && !entry.IsDirectory {
- fmt.Fprintf(writer, "block:%4d\tbyte:%10d\t%s/%s\n", blockCount, byteCount, dir, name)
+ numBlock, numByte, err := duTraverseDirectory(ctx, writer, filerClient, subDir, "")
+ if err == nil {
+ blockCount += numBlock
+ byteCount += numByte
}
+ } else {
+ blockCount += uint64(len(entry.Chunks))
+ byteCount += filer2.TotalSize(entry.Chunks)
}
- }
-
- if name == "" {
- fmt.Fprintf(writer, "block:%4d\tbyte:%10d\t%s\n", blockCount, byteCount, dir)
- }
+ if name != "" && !entry.IsDirectory {
+ fmt.Fprintf(writer, "block:%4d\tbyte:%10d\t%s/%s\n", blockCount, byteCount, dir, name)
+ }
+ })
return
-
}
func (env *CommandEnv) withFilerClient(ctx context.Context, filerServer string, filerPort int64, fn func(filer_pb.SeaweedFilerClient) error) error {
@@ -115,3 +92,20 @@ func (env *CommandEnv) withFilerClient(ctx context.Context, filerServer string,
}, filerGrpcAddress, env.option.GrpcDialOption)
}
+
+type commandFilerClient struct {
+ env *CommandEnv
+ filerServer string
+ filerPort int64
+}
+
+func (env *CommandEnv) getFilerClient(filerServer string, filerPort int64) *commandFilerClient {
+ return &commandFilerClient{
+ env: env,
+ filerServer: filerServer,
+ filerPort: filerPort,
+ }
+}
+func (c *commandFilerClient) WithFilerClient(ctx context.Context, fn func(filer_pb.SeaweedFilerClient) error) error {
+ return c.env.withFilerClient(ctx, c.filerServer, c.filerPort, fn)
+}
diff --git a/weed/shell/command_fs_ls.go b/weed/shell/command_fs_ls.go
index 6979635e1..01842083b 100644
--- a/weed/shell/command_fs_ls.go
+++ b/weed/shell/command_fs_ls.go
@@ -3,13 +3,14 @@ package shell
import (
"context"
"fmt"
- "github.com/chrislusf/seaweedfs/weed/filer2"
- "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"io"
"os"
"os/user"
"strconv"
"strings"
+
+ "github.com/chrislusf/seaweedfs/weed/filer2"
+ "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
)
func init() {
@@ -66,83 +67,51 @@ func (c *commandFsLs) Do(args []string, commandEnv *CommandEnv, writer io.Writer
}
dir, name := filer2.FullPath(path).DirAndName()
+ entryCount := 0
- return commandEnv.withFilerClient(ctx, filerServer, filerPort, func(client filer_pb.SeaweedFilerClient) error {
-
- return paginateOneDirectory(ctx, writer, client, dir, name, 1000, isLongFormat, showHidden)
-
- })
-
-}
-
-func paginateOneDirectory(ctx context.Context, writer io.Writer, client filer_pb.SeaweedFilerClient, dir, name string, paginateSize int, isLongFormat, showHidden bool) (err error) {
+ err = filer2.ReadDirAllEntries(ctx, commandEnv.getFilerClient(filerServer, filerPort), dir, name, func(entry *filer_pb.Entry, isLast bool) {
- entryCount := 0
- paginatedCount := -1
- startFromFileName := ""
-
- for paginatedCount == -1 || paginatedCount == paginateSize {
- resp, listErr := client.ListEntries(ctx, &filer_pb.ListEntriesRequest{
- Directory: dir,
- Prefix: name,
- StartFromFileName: startFromFileName,
- InclusiveStartFrom: false,
- Limit: uint32(paginateSize),
- })
- if listErr != nil {
- err = listErr
+ if !showHidden && strings.HasPrefix(entry.Name, ".") {
return
}
- paginatedCount = len(resp.Entries)
-
- for _, entry := range resp.Entries {
+ entryCount++
- if !showHidden && strings.HasPrefix(entry.Name, ".") {
- continue
- }
-
- entryCount++
-
- if isLongFormat {
- fileMode := os.FileMode(entry.Attributes.FileMode)
- userName, groupNames := entry.Attributes.UserName, entry.Attributes.GroupName
- if userName == "" {
- if user, userErr := user.LookupId(strconv.Itoa(int(entry.Attributes.Uid))); userErr == nil {
- userName = user.Username
- }
- }
- groupName := ""
- if len(groupNames) > 0 {
- groupName = groupNames[0]
+ if isLongFormat {
+ fileMode := os.FileMode(entry.Attributes.FileMode)
+ userName, groupNames := entry.Attributes.UserName, entry.Attributes.GroupName
+ if userName == "" {
+ if user, userErr := user.LookupId(strconv.Itoa(int(entry.Attributes.Uid))); userErr == nil {
+ userName = user.Username
}
- if groupName == "" {
- if group, groupErr := user.LookupGroupId(strconv.Itoa(int(entry.Attributes.Gid))); groupErr == nil {
- groupName = group.Name
- }
- }
-
- if dir == "/" {
- // just for printing
- dir = ""
+ }
+ groupName := ""
+ if len(groupNames) > 0 {
+ groupName = groupNames[0]
+ }
+ if groupName == "" {
+ if group, groupErr := user.LookupGroupId(strconv.Itoa(int(entry.Attributes.Gid))); groupErr == nil {
+ groupName = group.Name
}
- fmt.Fprintf(writer, "%s %3d %s %s %6d %s/%s\n",
- fileMode, len(entry.Chunks),
- userName, groupName,
- filer2.TotalSize(entry.Chunks), dir, entry.Name)
- } else {
- fmt.Fprintf(writer, "%s\n", entry.Name)
}
- startFromFileName = entry.Name
-
+ if dir == "/" {
+ // just for printing
+ dir = ""
+ }
+ fmt.Fprintf(writer, "%s %3d %s %s %6d %s/%s\n",
+ fileMode, len(entry.Chunks),
+ userName, groupName,
+ filer2.TotalSize(entry.Chunks), dir, entry.Name)
+ } else {
+ fmt.Fprintf(writer, "%s\n", entry.Name)
}
- }
- if isLongFormat {
+ })
+
+ if isLongFormat && err == nil {
fmt.Fprintf(writer, "total %d\n", entryCount)
}
return
-
}
diff --git a/weed/shell/command_fs_meta_notify.go b/weed/shell/command_fs_meta_notify.go
index 4fe0e45a9..a898df7a0 100644
--- a/weed/shell/command_fs_meta_notify.go
+++ b/weed/shell/command_fs_meta_notify.go
@@ -5,11 +5,12 @@ import (
"fmt"
"io"
+ "github.com/spf13/viper"
+
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/notification"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
- "github.com/spf13/viper"
)
func init() {
@@ -46,33 +47,33 @@ func (c *commandFsMetaNotify) Do(args []string, commandEnv *CommandEnv, writer i
ctx := context.Background()
- return commandEnv.withFilerClient(ctx, filerServer, filerPort, func(client filer_pb.SeaweedFilerClient) error {
+ var dirCount, fileCount uint64
- var dirCount, fileCount uint64
+ err = doTraverseBFS(ctx, writer, commandEnv.getFilerClient(filerServer, filerPort), filer2.FullPath(path), func(parentPath filer2.FullPath, entry *filer_pb.Entry) {
- err = doTraverseBFS(ctx, writer, client, filer2.FullPath(path), func(parentPath filer2.FullPath, entry *filer_pb.Entry) error {
-
- if entry.IsDirectory {
- dirCount++
- } else {
- fileCount++
- }
-
- return notification.Queue.SendMessage(
- string(parentPath.Child(entry.Name)),
- &filer_pb.EventNotification{
- NewEntry: entry,
- },
- )
+ if entry.IsDirectory {
+ dirCount++
+ } else {
+ fileCount++
+ }
- })
+ notifyErr := notification.Queue.SendMessage(
+ string(parentPath.Child(entry.Name)),
+ &filer_pb.EventNotification{
+ NewEntry: entry,
+ },
+ )
- if err == nil {
- fmt.Fprintf(writer, "\ntotal notified %d directories, %d files\n", dirCount, fileCount)
+ if notifyErr != nil {
+ fmt.Fprintf(writer, "fail to notify new entry event for %s: %v\n", parentPath.Child(entry.Name), notifyErr)
}
- return err
-
})
+ if err == nil {
+ fmt.Fprintf(writer, "\ntotal notified %d directories, %d files\n", dirCount, fileCount)
+ }
+
+ return err
+
}
diff --git a/weed/shell/command_fs_meta_save.go b/weed/shell/command_fs_meta_save.go
index dd5e9defb..4ff00c64b 100644
--- a/weed/shell/command_fs_meta_save.go
+++ b/weed/shell/command_fs_meta_save.go
@@ -54,74 +54,69 @@ func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io.
return nil
}
- filerServer, filerPort, path, err := commandEnv.parseUrl(findInputDirectory(fsMetaSaveCommand.Args()))
- if err != nil {
- return err
+ filerServer, filerPort, path, parseErr := commandEnv.parseUrl(findInputDirectory(fsMetaSaveCommand.Args()))
+ if parseErr != nil {
+ return parseErr
}
ctx := context.Background()
- return commandEnv.withFilerClient(ctx, filerServer, filerPort, func(client filer_pb.SeaweedFilerClient) error {
-
- t := time.Now()
- fileName := *outputFileName
- if fileName == "" {
- fileName = fmt.Sprintf("%s-%d-%4d%02d%02d-%02d%02d%02d.meta",
- filerServer, filerPort, t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second())
- }
-
- dst, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
- if err != nil {
- return nil
- }
- defer dst.Close()
-
- var dirCount, fileCount uint64
-
- err = doTraverseBFS(ctx, writer, client, filer2.FullPath(path), func(parentPath filer2.FullPath, entry *filer_pb.Entry) error {
-
- protoMessage := &filer_pb.FullEntry{
- Dir: string(parentPath),
- Entry: entry,
- }
+ t := time.Now()
+ fileName := *outputFileName
+ if fileName == "" {
+ fileName = fmt.Sprintf("%s-%d-%4d%02d%02d-%02d%02d%02d.meta",
+ filerServer, filerPort, t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second())
+ }
- bytes, err := proto.Marshal(protoMessage)
- if err != nil {
- return fmt.Errorf("marshall error: %v", err)
- }
+ dst, openErr := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+ if openErr != nil {
+ return fmt.Errorf("failed to create file %s: %v", fileName, openErr)
+ }
+ defer dst.Close()
- sizeBuf := make([]byte, 4)
- util.Uint32toBytes(sizeBuf, uint32(len(bytes)))
+ var dirCount, fileCount uint64
- dst.Write(sizeBuf)
- dst.Write(bytes)
+ err = doTraverseBFS(ctx, writer, commandEnv.getFilerClient(filerServer, filerPort), filer2.FullPath(path), func(parentPath filer2.FullPath, entry *filer_pb.Entry) {
- if entry.IsDirectory {
- atomic.AddUint64(&dirCount, 1)
- } else {
- atomic.AddUint64(&fileCount, 1)
- }
+ protoMessage := &filer_pb.FullEntry{
+ Dir: string(parentPath),
+ Entry: entry,
+ }
- if *verbose {
- println(parentPath.Child(entry.Name))
- }
+ bytes, err := proto.Marshal(protoMessage)
+ if err != nil {
+ fmt.Fprintf(writer, "marshall error: %v\n", err)
+ return
+ }
- return nil
+ sizeBuf := make([]byte, 4)
+ util.Uint32toBytes(sizeBuf, uint32(len(bytes)))
- })
+ dst.Write(sizeBuf)
+ dst.Write(bytes)
- if err == nil {
- fmt.Fprintf(writer, "\ntotal %d directories, %d files", dirCount, fileCount)
- fmt.Fprintf(writer, "\nmeta data for http://%s:%d%s is saved to %s\n", filerServer, filerPort, path, fileName)
+ if entry.IsDirectory {
+ atomic.AddUint64(&dirCount, 1)
+ } else {
+ atomic.AddUint64(&fileCount, 1)
}
- return err
+ if *verbose {
+ println(parentPath.Child(entry.Name))
+ }
})
+ if err == nil {
+ fmt.Fprintf(writer, "\ntotal %d directories, %d files", dirCount, fileCount)
+ fmt.Fprintf(writer, "\nmeta data for http://%s:%d%s is saved to %s\n", filerServer, filerPort, path, fileName)
+ }
+
+ return err
+
}
-func doTraverseBFS(ctx context.Context, writer io.Writer, client filer_pb.SeaweedFilerClient,
- parentPath filer2.FullPath, fn func(parentPath filer2.FullPath, entry *filer_pb.Entry) error) (err error) {
+func doTraverseBFS(ctx context.Context, writer io.Writer, filerClient filer2.FilerClient,
+ parentPath filer2.FullPath, fn func(parentPath filer2.FullPath, entry *filer_pb.Entry)) (err error) {
K := 5
@@ -143,7 +138,7 @@ func doTraverseBFS(ctx context.Context, writer io.Writer, client filer_pb.Seawee
continue
}
dir := t.(filer2.FullPath)
- processErr := processOneDirectory(ctx, writer, client, dir, queue, &jobQueueWg, fn)
+ processErr := processOneDirectory(ctx, writer, filerClient, dir, queue, &jobQueueWg, fn)
if processErr != nil {
err = processErr
}
@@ -156,47 +151,22 @@ func doTraverseBFS(ctx context.Context, writer io.Writer, client filer_pb.Seawee
return
}
-func processOneDirectory(ctx context.Context, writer io.Writer, client filer_pb.SeaweedFilerClient,
+func processOneDirectory(ctx context.Context, writer io.Writer, filerClient filer2.FilerClient,
parentPath filer2.FullPath, queue *util.Queue, jobQueueWg *sync.WaitGroup,
- fn func(parentPath filer2.FullPath, entry *filer_pb.Entry) error) (err error) {
-
- paginatedCount := -1
- startFromFileName := ""
- paginateSize := 1000
-
- for paginatedCount == -1 || paginatedCount == paginateSize {
- resp, listErr := client.ListEntries(ctx, &filer_pb.ListEntriesRequest{
- Directory: string(parentPath),
- Prefix: "",
- StartFromFileName: startFromFileName,
- InclusiveStartFrom: false,
- Limit: uint32(paginateSize),
- })
- if listErr != nil {
- err = listErr
- return
- }
+ fn func(parentPath filer2.FullPath, entry *filer_pb.Entry)) (err error) {
- paginatedCount = len(resp.Entries)
+ return filer2.ReadDirAllEntries(ctx, filerClient, string(parentPath), "", func(entry *filer_pb.Entry, isLast bool) {
- for _, entry := range resp.Entries {
+ fn(parentPath, entry)
- if err = fn(parentPath, entry); err != nil {
- return err
+ if entry.IsDirectory {
+ subDir := fmt.Sprintf("%s/%s", parentPath, entry.Name)
+ if parentPath == "/" {
+ subDir = "/" + entry.Name
}
-
- if entry.IsDirectory {
- subDir := fmt.Sprintf("%s/%s", parentPath, entry.Name)
- if parentPath == "/" {
- subDir = "/" + entry.Name
- }
- jobQueueWg.Add(1)
- queue.Enqueue(filer2.FullPath(subDir))
- }
- startFromFileName = entry.Name
+ jobQueueWg.Add(1)
+ queue.Enqueue(filer2.FullPath(subDir))
}
- }
-
- return
+ })
}
diff --git a/weed/shell/command_fs_tree.go b/weed/shell/command_fs_tree.go
index 8474e43ea..a4524f341 100644
--- a/weed/shell/command_fs_tree.go
+++ b/weed/shell/command_fs_tree.go
@@ -3,10 +3,11 @@ package shell
import (
"context"
"fmt"
- "github.com/chrislusf/seaweedfs/weed/filer2"
- "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"io"
"strings"
+
+ "github.com/chrislusf/seaweedfs/weed/filer2"
+ "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
)
func init() {
@@ -38,75 +39,45 @@ func (c *commandFsTree) Do(args []string, commandEnv *CommandEnv, writer io.Writ
ctx := context.Background()
- return commandEnv.withFilerClient(ctx, filerServer, filerPort, func(client filer_pb.SeaweedFilerClient) error {
+ dirCount, fCount, terr := treeTraverseDirectory(ctx, writer, commandEnv.getFilerClient(filerServer, filerPort), dir, name, newPrefix(), -1)
- dirCount, fCount, terr := treeTraverseDirectory(ctx, writer, client, dir, name, newPrefix(), -1)
-
- if terr == nil {
- fmt.Fprintf(writer, "%d directories, %d files\n", dirCount, fCount)
- }
-
- return terr
+ if terr == nil {
+ fmt.Fprintf(writer, "%d directories, %d files\n", dirCount, fCount)
+ }
- })
+ return terr
}
-func treeTraverseDirectory(ctx context.Context, writer io.Writer, client filer_pb.SeaweedFilerClient, dir, name string, prefix *Prefix, level int) (directoryCount, fileCount int64, err error) {
-
- paginatedCount := -1
- startFromFileName := ""
- paginateSize := 1000
-
- for paginatedCount == -1 || paginatedCount == paginateSize {
- resp, listErr := client.ListEntries(ctx, &filer_pb.ListEntriesRequest{
- Directory: dir,
- Prefix: name,
- StartFromFileName: startFromFileName,
- InclusiveStartFrom: false,
- Limit: uint32(paginateSize),
- })
- if listErr != nil {
- err = listErr
- return
- }
- paginatedCount = len(resp.Entries)
- if paginatedCount > 0 {
- prefix.addMarker(level)
- }
+func treeTraverseDirectory(ctx context.Context, writer io.Writer, filerClient filer2.FilerClient, dir, name string, prefix *Prefix, level int) (directoryCount, fileCount int64, err error) {
- for i, entry := range resp.Entries {
+ prefix.addMarker(level)
- if level < 0 && name != "" {
- if entry.Name != name {
- break
- }
+ err = filer2.ReadDirAllEntries(ctx, filerClient, dir, name, func(entry *filer_pb.Entry, isLast bool) {
+ if level < 0 && name != "" {
+ if entry.Name != name {
+ return
}
+ }
- // 0.1% wrong prefix here, but fixing it would need to paginate to the next batch first
- isLast := paginatedCount < paginateSize && i == paginatedCount-1
- fmt.Fprintf(writer, "%s%s\n", prefix.getPrefix(level, isLast), entry.Name)
-
- if entry.IsDirectory {
- directoryCount++
- subDir := fmt.Sprintf("%s/%s", dir, entry.Name)
- if dir == "/" {
- subDir = "/" + entry.Name
- }
- dirCount, fCount, terr := treeTraverseDirectory(ctx, writer, client, subDir, "", prefix, level+1)
- directoryCount += dirCount
- fileCount += fCount
- err = terr
- } else {
- fileCount++
- }
- startFromFileName = entry.Name
+ fmt.Fprintf(writer, "%s%s\n", prefix.getPrefix(level, isLast), entry.Name)
+ if entry.IsDirectory {
+ directoryCount++
+ subDir := fmt.Sprintf("%s/%s", dir, entry.Name)
+ if dir == "/" {
+ subDir = "/" + entry.Name
+ }
+ dirCount, fCount, terr := treeTraverseDirectory(ctx, writer, filerClient, subDir, "", prefix, level+1)
+ directoryCount += dirCount
+ fileCount += fCount
+ err = terr
+ } else {
+ fileCount++
}
- }
+ })
return
-
}
type Prefix struct {
diff --git a/weed/shell/commands.go b/weed/shell/commands.go
index b642ec253..a6a0f7dec 100644
--- a/weed/shell/commands.go
+++ b/weed/shell/commands.go
@@ -9,10 +9,11 @@ import (
"strconv"
"strings"
+ "google.golang.org/grpc"
+
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/wdclient"
- "google.golang.org/grpc"
)
type ShellOptions struct {
@@ -71,26 +72,19 @@ func (ce *CommandEnv) checkDirectory(ctx context.Context, filerServer string, fi
return ce.withFilerClient(ctx, filerServer, filerPort, func(client filer_pb.SeaweedFilerClient) error {
- resp, listErr := client.ListEntries(ctx, &filer_pb.ListEntriesRequest{
- Directory: dir,
- Prefix: name,
- StartFromFileName: name,
- InclusiveStartFrom: true,
- Limit: 1,
+ resp, lookupErr := client.LookupDirectoryEntry(ctx, &filer_pb.LookupDirectoryEntryRequest{
+ Directory: dir,
+ Name: name,
})
- if listErr != nil {
- return listErr
+ if lookupErr != nil {
+ return lookupErr
}
- if len(resp.Entries) == 0 {
+ if resp.Entry == nil {
return fmt.Errorf("entry not found")
}
- if resp.Entries[0].Name != name {
- return fmt.Errorf("not a valid directory, found %s", resp.Entries[0].Name)
- }
-
- if !resp.Entries[0].IsDirectory {
+ if !resp.Entry.IsDirectory {
return fmt.Errorf("not a directory")
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-21 14:27:00 +0000