diff options
Diffstat (limited to 'weed')
48 files changed, 1176 insertions, 246 deletions
diff --git a/weed/command/master.go b/weed/command/master.go index 3d33f4f7a..8d0a3289c 100644 --- a/weed/command/master.go +++ b/weed/command/master.go @@ -12,6 +12,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/server" + "github.com/chrislusf/seaweedfs/weed/storage/backend" "github.com/chrislusf/seaweedfs/weed/util" "github.com/gorilla/mux" "github.com/spf13/viper" @@ -101,6 +102,8 @@ func runMaster(cmd *Command, args []string) bool { func startMaster(masterOption MasterOptions, masterWhiteList []string) { + backend.LoadConfiguration(viper.GetViper()) + myMasterAddress, peers := checkPeers(*masterOption.ip, *masterOption.port, *masterOption.peers) r := mux.NewRouter() diff --git a/weed/command/mount.go b/weed/command/mount.go index 71c1a4387..c6c346271 100644 --- a/weed/command/mount.go +++ b/weed/command/mount.go @@ -64,12 +64,12 @@ var cmdMount = &Command{ func parseFilerGrpcAddress(filer string) (filerGrpcAddress string, err error) { hostnameAndPort := strings.Split(filer, ":") if len(hostnameAndPort) != 2 { - return "", fmt.Errorf("The filer should have hostname:port format: %v", hostnameAndPort) + return "", fmt.Errorf("filer should have hostname:port format: %v", hostnameAndPort) } filerPort, parseErr := strconv.ParseUint(hostnameAndPort[1], 10, 64) if parseErr != nil { - return "", fmt.Errorf("The filer filer port parse error: %v", parseErr) + return "", fmt.Errorf("filer port parse error: %v", parseErr) } filerGrpcPort := int(filerPort) + 10000 diff --git a/weed/command/mount_darwin.go b/weed/command/mount_darwin.go index 632691e47..f0a5581e7 100644 --- a/weed/command/mount_darwin.go +++ b/weed/command/mount_darwin.go @@ -7,3 +7,7 @@ import ( func osSpecificMountOptions() []fuse.MountOption { return []fuse.MountOption{} } + +func checkMountPointAvailable(dir string) bool { + return true +} diff --git a/weed/command/mount_freebsd.go b/weed/command/mount_freebsd.go index 632691e47..f0a5581e7 100644 --- a/weed/command/mount_freebsd.go +++ b/weed/command/mount_freebsd.go @@ -7,3 +7,7 @@ import ( func osSpecificMountOptions() []fuse.MountOption { return []fuse.MountOption{} } + +func checkMountPointAvailable(dir string) bool { + return true +} diff --git a/weed/command/mount_linux.go b/weed/command/mount_linux.go index 7d94e5142..80a5f9da4 100644 --- a/weed/command/mount_linux.go +++ b/weed/command/mount_linux.go @@ -1,11 +1,157 @@ package command import ( + "bufio" + "fmt" + "io" + "os" + "strings" + "github.com/seaweedfs/fuse" ) +const ( + /* 36 35 98:0 /mnt1 /mnt2 rw,noatime master:1 - ext3 /dev/root rw,errors=continue + (1)(2)(3) (4) (5) (6) (7) (8) (9) (10) (11) + + (1) mount ID: unique identifier of the mount (may be reused after umount) + (2) parent ID: ID of parent (or of self for the top of the mount tree) + (3) major:minor: value of st_dev for files on filesystem + (4) root: root of the mount within the filesystem + (5) mount point: mount point relative to the process's root + (6) mount options: per mount options + (7) optional fields: zero or more fields of the form "tag[:value]" + (8) separator: marks the end of the optional fields + (9) filesystem type: name of filesystem of the form "type[.subtype]" + (10) mount source: filesystem specific information or "none" + (11) super options: per super block options*/ + mountinfoFormat = "%d %d %d:%d %s %s %s %s" +) + +// Info reveals information about a particular mounted filesystem. This +// struct is populated from the content in the /proc/<pid>/mountinfo file. +type Info struct { + // ID is a unique identifier of the mount (may be reused after umount). + ID int + + // Parent indicates the ID of the mount parent (or of self for the top of the + // mount tree). + Parent int + + // Major indicates one half of the device ID which identifies the device class. + Major int + + // Minor indicates one half of the device ID which identifies a specific + // instance of device. + Minor int + + // Root of the mount within the filesystem. + Root string + + // Mountpoint indicates the mount point relative to the process's root. + Mountpoint string + + // Opts represents mount-specific options. + Opts string + + // Optional represents optional fields. + Optional string + + // Fstype indicates the type of filesystem, such as EXT3. + Fstype string + + // Source indicates filesystem specific information or "none". + Source string + + // VfsOpts represents per super block options. + VfsOpts string +} + +// Mounted determines if a specified mountpoint has been mounted. +// On Linux it looks at /proc/self/mountinfo and on Solaris at mnttab. +func mounted(mountPoint string) (bool, error) { + entries, err := parseMountTable() + if err != nil { + return false, err + } + + // Search the table for the mountPoint + for _, e := range entries { + if e.Mountpoint == mountPoint { + return true, nil + } + } + return false, nil +} + +// Parse /proc/self/mountinfo because comparing Dev and ino does not work from +// bind mounts +func parseMountTable() ([]*Info, error) { + f, err := os.Open("/proc/self/mountinfo") + if err != nil { + return nil, err + } + defer f.Close() + + return parseInfoFile(f) +} + +func parseInfoFile(r io.Reader) ([]*Info, error) { + var ( + s = bufio.NewScanner(r) + out []*Info + ) + + for s.Scan() { + if err := s.Err(); err != nil { + return nil, err + } + + var ( + p = &Info{} + text = s.Text() + optionalFields string + ) + + if _, err := fmt.Sscanf(text, mountinfoFormat, + &p.ID, &p.Parent, &p.Major, &p.Minor, + &p.Root, &p.Mountpoint, &p.Opts, &optionalFields); err != nil { + return nil, fmt.Errorf("Scanning '%s' failed: %s", text, err) + } + // Safe as mountinfo encodes mountpoints with spaces as \040. + index := strings.Index(text, " - ") + postSeparatorFields := strings.Fields(text[index+3:]) + if len(postSeparatorFields) < 3 { + return nil, fmt.Errorf("Error found less than 3 fields post '-' in %q", text) + } + + if optionalFields != "-" { + p.Optional = optionalFields + } + + p.Fstype = postSeparatorFields[0] + p.Source = postSeparatorFields[1] + p.VfsOpts = strings.Join(postSeparatorFields[2:], " ") + out = append(out, p) + } + return out, nil +} + func osSpecificMountOptions() []fuse.MountOption { return []fuse.MountOption{ fuse.AllowNonEmptyMount(), } } + +func checkMountPointAvailable(dir string) bool { + mountPoint := dir + if mountPoint != "/" && strings.HasSuffix(mountPoint, "/") { + mountPoint = mountPoint[0 : len(mountPoint)-1] + } + + if mounted, err := mounted(mountPoint); err != nil || mounted { + return false + } + + return true +} diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index 6ca9bfdca..9fde502d6 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -12,15 +12,14 @@ import ( "strings" "time" - "github.com/chrislusf/seaweedfs/weed/security" - "github.com/jacobsa/daemonize" - "github.com/spf13/viper" - "github.com/chrislusf/seaweedfs/weed/filesys" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/util" + "github.com/jacobsa/daemonize" "github.com/seaweedfs/fuse" "github.com/seaweedfs/fuse/fs" + "github.com/spf13/viper" ) func runMount(cmd *Command, args []string) bool { @@ -88,12 +87,18 @@ func RunMount(filer, filerMountRootPath, dir, collection, replication, dataCente } } + // Ensure target mount point availability + if isValid := checkMountPointAvailable(dir); !isValid { + glog.Fatalf("Expected mount to still be active, target mount point: %s, please check!", dir) + return false + } + mountName := path.Base(dir) options := []fuse.MountOption{ fuse.VolumeName(mountName), - fuse.FSName("SeaweedFS"), - fuse.Subtype("SeaweedFS"), + fuse.FSName(filer + ":" + filerMountRootPath), + fuse.Subtype("seaweedfs"), fuse.NoAppleDouble(), fuse.NoAppleXattr(), fuse.NoBrowse(), diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index 76b244a05..667324a8a 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -95,6 +95,7 @@ password = "" database = "" # create or use an existing database connection_max_idle = 2 connection_max_open = 100 +interpolateParams = false [postgres] # or cockroachdb # CREATE TABLE IF NOT EXISTS filemeta ( @@ -346,14 +347,24 @@ scripts = """ """ sleep_minutes = 17 # sleep minutes between each script execution -filer_url = "http://localhost:8888/" - -sequencer_type = memory # Choose [memory|etcd] type for storing the file id sequence +[master.filer] +default_filer_url = "http://localhost:8888/" +[master.sequencer] +type = "memory" # Choose [memory|etcd] type for storing the file id sequence # when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence # example : http://127.0.0.1:2379,http://127.0.0.1:2389 -sequencer_etcd_urls = http://127.0.0.1:2379 +sequencer_etcd_urls = "http://127.0.0.1:2379" + +[storage.backend] + [storage.backend.s3.default] + enabled = false + aws_access_key_id = "" # if empty, loads from the shared credentials file (~/.aws/credentials). + aws_secret_access_key = "" # if empty, loads from the shared credentials file (~/.aws/credentials). + region = "us-east-2" + bucket = "your_bucket_name" # an existing bucket + directory = "/" # destination directory ` ) diff --git a/weed/command/scaffold_test.go b/weed/command/scaffold_test.go new file mode 100644 index 000000000..423dacc32 --- /dev/null +++ b/weed/command/scaffold_test.go @@ -0,0 +1,44 @@ +package command + +import ( + "bytes" + "fmt" + "testing" + + "github.com/spf13/viper" +) + +func TestReadingTomlConfiguration(t *testing.T) { + + viper.SetConfigType("toml") + + // any approach to require this configuration into your program. + var tomlExample = []byte(` +[database] +server = "192.168.1.1" +ports = [ 8001, 8001, 8002 ] +connection_max = 5000 +enabled = true + +[servers] + + # You can indent as you please. Tabs or spaces. TOML don't care. + [servers.alpha] + ip = "10.0.0.1" + dc = "eqdc10" + + [servers.beta] + ip = "10.0.0.2" + dc = "eqdc10" + +`) + + viper.ReadConfig(bytes.NewBuffer(tomlExample)) + + fmt.Printf("database is %v\n", viper.Get("database")) + fmt.Printf("servers is %v\n", viper.GetStringMap("servers")) + + alpha := viper.Sub("servers.alpha") + + fmt.Printf("alpha ip is %v\n", alpha.GetString("ip")) +} diff --git a/weed/filer2/mysql/mysql_store.go b/weed/filer2/mysql/mysql_store.go index e18299bd2..3eca80ff7 100644 --- a/weed/filer2/mysql/mysql_store.go +++ b/weed/filer2/mysql/mysql_store.go @@ -35,10 +35,12 @@ func (store *MysqlStore) Initialize(configuration util.Configuration) (err error configuration.GetString("database"), configuration.GetInt("connection_max_idle"), configuration.GetInt("connection_max_open"), + configuration.GetBool("interpolateParams"), ) } -func (store *MysqlStore) initialize(user, password, hostname string, port int, database string, maxIdle, maxOpen int) (err error) { +func (store *MysqlStore) initialize(user, password, hostname string, port int, database string, maxIdle, maxOpen int, + interpolateParams bool) (err error) { store.SqlInsert = "INSERT INTO filemeta (dirhash,name,directory,meta) VALUES(?,?,?,?)" store.SqlUpdate = "UPDATE filemeta SET meta=? WHERE dirhash=? AND name=? AND directory=?" @@ -48,6 +50,10 @@ func (store *MysqlStore) initialize(user, password, hostname string, port int, d store.SqlListInclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>=? AND directory=? ORDER BY NAME ASC LIMIT ?" sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database) + if interpolateParams { + sqlUrl += "&interpolateParams=true" + } + var dbErr error store.DB, dbErr = sql.Open("mysql", sqlUrl) if dbErr != nil { diff --git a/weed/filer2/tikv/tikv_store.go b/weed/filer2/tikv/tikv_store.go index 8143130ab..0e6b93c86 100644 --- a/weed/filer2/tikv/tikv_store.go +++ b/weed/filer2/tikv/tikv_store.go @@ -1,3 +1,6 @@ +// +build !386 +// +build !arm + package tikv import ( diff --git a/weed/filer2/tikv/tikv_store_unsupported.go b/weed/filer2/tikv/tikv_store_unsupported.go new file mode 100644 index 000000000..95c8b2dad --- /dev/null +++ b/weed/filer2/tikv/tikv_store_unsupported.go @@ -0,0 +1,61 @@ +// +build 386 arm + +package tikv + +import ( + "context" + "fmt" + + "github.com/chrislusf/seaweedfs/weed/filer2" + weed_util "github.com/chrislusf/seaweedfs/weed/util" +) + +func init() { + filer2.Stores = append(filer2.Stores, &TikvStore{}) +} + +type TikvStore struct { +} + +func (store *TikvStore) GetName() string { + return "tikv" +} + +func (store *TikvStore) Initialize(configuration weed_util.Configuration) (err error) { + return fmt.Errorf("not implemented for 32 bit computers") +} + +func (store *TikvStore) initialize(pdAddr string) (err error) { + return fmt.Errorf("not implemented for 32 bit computers") +} + +func (store *TikvStore) BeginTransaction(ctx context.Context) (context.Context, error) { + return nil, fmt.Errorf("not implemented for 32 bit computers") +} +func (store *TikvStore) CommitTransaction(ctx context.Context) error { + return fmt.Errorf("not implemented for 32 bit computers") +} +func (store *TikvStore) RollbackTransaction(ctx context.Context) error { + return fmt.Errorf("not implemented for 32 bit computers") +} + +func (store *TikvStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) { + return fmt.Errorf("not implemented for 32 bit computers") +} + +func (store *TikvStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) { + return fmt.Errorf("not implemented for 32 bit computers") +} + +func (store *TikvStore) FindEntry(ctx context.Context, fullpath filer2.FullPath) (entry *filer2.Entry, err error) { + return nil, fmt.Errorf("not implemented for 32 bit computers") +} + +func (store *TikvStore) DeleteEntry(ctx context.Context, fullpath filer2.FullPath) (err error) { + return fmt.Errorf("not implemented for 32 bit computers") +} + +func (store *TikvStore) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool, + limit int) (entries []*filer2.Entry, err error) { + return nil, fmt.Errorf("not implemented for 32 bit computers") +} diff --git a/weed/notification/aws_sqs/aws_sqs_pub.go b/weed/notification/aws_sqs/aws_sqs_pub.go index c1af7f27a..4c1302abb 100644 --- a/weed/notification/aws_sqs/aws_sqs_pub.go +++ b/weed/notification/aws_sqs/aws_sqs_pub.go @@ -38,13 +38,13 @@ func (k *AwsSqsPub) Initialize(configuration util.Configuration) (err error) { ) } -func (k *AwsSqsPub) initialize(awsAccessKeyId, aswSecretAccessKey, region, queueName string) (err error) { +func (k *AwsSqsPub) initialize(awsAccessKeyId, awsSecretAccessKey, region, queueName string) (err error) { config := &aws.Config{ Region: aws.String(region), } - if awsAccessKeyId != "" && aswSecretAccessKey != "" { - config.Credentials = credentials.NewStaticCredentials(awsAccessKeyId, aswSecretAccessKey, "") + if awsAccessKeyId != "" && awsSecretAccessKey != "" { + config.Credentials = credentials.NewStaticCredentials(awsAccessKeyId, awsSecretAccessKey, "") } sess, err := session.NewSession(config) diff --git a/weed/notification/kafka/kafka_queue.go b/weed/notification/kafka/kafka_queue.go index 830709a51..fd545722b 100644 --- a/weed/notification/kafka/kafka_queue.go +++ b/weed/notification/kafka/kafka_queue.go @@ -76,7 +76,7 @@ func (k *KafkaQueue) handleError() { for { err := <-k.producer.Errors() if err != nil { - glog.Errorf("producer message error, partition:%d offset:%d key:%v valus:%s error(%v) topic:%s", err.Msg.Partition, err.Msg.Offset, err.Msg.Key, err.Msg.Value, err.Err, k.topic) + glog.Errorf("producer message error, partition:%d offset:%d key:%v value:%s error(%v) topic:%s", err.Msg.Partition, err.Msg.Offset, err.Msg.Key, err.Msg.Value, err.Err, k.topic) } } } diff --git a/weed/pb/proto_read_write_test.go b/weed/pb/proto_read_write_test.go new file mode 100644 index 000000000..4c53ffa2f --- /dev/null +++ b/weed/pb/proto_read_write_test.go @@ -0,0 +1,43 @@ +package pb + +import ( + "fmt" + "testing" + + "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" + "github.com/golang/protobuf/jsonpb" +) + +func TestJsonpMarshalUnmarshal(t *testing.T) { + + tv := &volume_server_pb.TieredVolume{ + BackendType: "aws", + BackendName: "", + Version: 12, + } + + m := jsonpb.Marshaler{ + EmitDefaults: true, + Indent: " ", + } + + if text, err := m.MarshalToString(tv); err != nil { + fmt.Printf("marshal eror: %v\n", err) + } else { + fmt.Printf("marshalled: %s\n", text) + } + + rawJson := `{ + "backendType":"aws", + "backendName":"temp", + "version":12 + }` + + tv1 := &volume_server_pb.TieredVolume{} + if err := jsonpb.UnmarshalString(rawJson, tv1); err != nil { + fmt.Printf("unmarshal eror: %v\n", err) + } + + fmt.Printf("unmarshalled: %+v\n", tv1) + +} diff --git a/weed/pb/volume_server.proto b/weed/pb/volume_server.proto index 3a5874c02..d30d61f71 100644 --- a/weed/pb/volume_server.proto +++ b/weed/pb/volume_server.proto @@ -67,6 +67,10 @@ service VolumeServer { rpc VolumeEcBlobDelete (VolumeEcBlobDeleteRequest) returns (VolumeEcBlobDeleteResponse) { } + // tiered storage + rpc VolumeTierCopyDatToRemote (VolumeTierCopyDatToRemoteRequest) returns (VolumeTierCopyDatToRemoteResponse) { + } + // query rpc Query (QueryRequest) returns (stream QueriedStripe) { } @@ -324,6 +328,22 @@ message MemStatus { uint64 stack = 7; } +// tired storage on volume servers +message TieredVolume { + string backend_type = 1; + string backend_name = 2; + uint64 version = 3; +} + +message VolumeTierCopyDatToRemoteRequest { + uint32 volume_id = 1; + string collection = 2; + string destination = 3; +} +message VolumeTierCopyDatToRemoteResponse { +} + +// select on volume servers message QueryRequest { repeated string selections = 1; repeated string from_file_ids = 2; diff --git a/weed/pb/volume_server_pb/volume_server.pb.go b/weed/pb/volume_server_pb/volume_server.pb.go index 24907cdba..684a8b08c 100644 --- a/weed/pb/volume_server_pb/volume_server.pb.go +++ b/weed/pb/volume_server_pb/volume_server.pb.go @@ -65,6 +65,9 @@ It has these top-level messages: ReadVolumeFileStatusResponse DiskStatus MemStatus + TieredVolume + VolumeTierCopyDatToRemoteRequest + VolumeTierCopyDatToRemoteResponse QueryRequest QueriedStripe */ @@ -1394,6 +1397,76 @@ func (m *MemStatus) GetStack() uint64 { return 0 } +// tired storage on volume servers +type TieredVolume struct { + BackendType string `protobuf:"bytes,1,opt,name=backend_type,json=backendType" json:"backend_type,omitempty"` + BackendName string `protobuf:"bytes,2,opt,name=backend_name,json=backendName" json:"backend_name,omitempty"` + Version uint64 `protobuf:"varint,3,opt,name=version" json:"version,omitempty"` +} + +func (m *TieredVolume) Reset() { *m = TieredVolume{} } +func (m *TieredVolume) String() string { return proto.CompactTextString(m) } +func (*TieredVolume) ProtoMessage() {} +func (*TieredVolume) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{56} } + +func (m *TieredVolume) GetBackendType() string { + if m != nil { + return m.BackendType + } + return "" +} + +func (m *TieredVolume) GetBackendName() string { + if m != nil { + return m.BackendName + } + return "" +} + +func (m *TieredVolume) GetVersion() uint64 { + if m != nil { + return m.Version + } + return 0 +} + +type VolumeTierCopyDatToRemoteRequest struct { + VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` + Collection string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"` +} + +func (m *VolumeTierCopyDatToRemoteRequest) Reset() { *m = VolumeTierCopyDatToRemoteRequest{} } +func (m *VolumeTierCopyDatToRemoteRequest) String() string { return proto.CompactTextString(m) } +func (*VolumeTierCopyDatToRemoteRequest) ProtoMessage() {} +func (*VolumeTierCopyDatToRemoteRequest) Descriptor() ([]byte, []int) { + return fileDescriptor0, []int{57} +} + +func (m *VolumeTierCopyDatToRemoteRequest) GetVolumeId() uint32 { + if m != nil { + return m.VolumeId + } + return 0 +} + +func (m *VolumeTierCopyDatToRemoteRequest) GetCollection() string { + if m != nil { + return m.Collection + } + return "" +} + +type VolumeTierCopyDatToRemoteResponse struct { +} + +func (m *VolumeTierCopyDatToRemoteResponse) Reset() { *m = VolumeTierCopyDatToRemoteResponse{} } +func (m *VolumeTierCopyDatToRemoteResponse) String() string { return proto.CompactTextString(m) } +func (*VolumeTierCopyDatToRemoteResponse) ProtoMessage() {} +func (*VolumeTierCopyDatToRemoteResponse) Descriptor() ([]byte, []int) { + return fileDescriptor0, []int{58} +} + +// select on volume servers type QueryRequest struct { Selections []string `protobuf:"bytes,1,rep,name=selections" json:"selections,omitempty"` FromFileIds []string `protobuf:"bytes,2,rep,name=from_file_ids,json=fromFileIds" json:"from_file_ids,omitempty"` @@ -1405,7 +1478,7 @@ type QueryRequest struct { func (m *QueryRequest) Reset() { *m = QueryRequest{} } func (m *QueryRequest) String() string { return proto.CompactTextString(m) } func (*QueryRequest) ProtoMessage() {} -func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{56} } +func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{59} } func (m *QueryRequest) GetSelections() []string { if m != nil { @@ -1451,7 +1524,7 @@ type QueryRequest_Filter struct { func (m *QueryRequest_Filter) Reset() { *m = QueryRequest_Filter{} } func (m *QueryRequest_Filter) String() string { return proto.CompactTextString(m) } func (*QueryRequest_Filter) ProtoMessage() {} -func (*QueryRequest_Filter) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{56, 0} } +func (*QueryRequest_Filter) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{59, 0} } func (m *QueryRequest_Filter) GetField() string { if m != nil { @@ -1486,7 +1559,7 @@ func (m *QueryRequest_InputSerialization) Reset() { *m = QueryRequest_In func (m *QueryRequest_InputSerialization) String() string { return proto.CompactTextString(m) } func (*QueryRequest_InputSerialization) ProtoMessage() {} func (*QueryRequest_InputSerialization) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{56, 1} + return fileDescriptor0, []int{59, 1} } func (m *QueryRequest_InputSerialization) GetCompressionType() string { @@ -1534,7 +1607,7 @@ func (m *QueryRequest_InputSerialization_CSVInput) Reset() { func (m *QueryRequest_InputSerialization_CSVInput) String() string { return proto.CompactTextString(m) } func (*QueryRequest_InputSerialization_CSVInput) ProtoMessage() {} func (*QueryRequest_InputSerialization_CSVInput) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{56, 1, 0} + return fileDescriptor0, []int{59, 1, 0} } func (m *QueryRequest_InputSerialization_CSVInput) GetFileHeaderInfo() string { @@ -1596,7 +1669,7 @@ func (m *QueryRequest_InputSerialization_JSONInput) Reset() { func (m *QueryRequest_InputSerialization_JSONInput) String() string { return proto.CompactTextString(m) } func (*QueryRequest_InputSerialization_JSONInput) ProtoMessage() {} func (*QueryRequest_InputSerialization_JSONInput) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{56, 1, 1} + return fileDescriptor0, []int{59, 1, 1} } func (m *QueryRequest_InputSerialization_JSONInput) GetType() string { @@ -1617,7 +1690,7 @@ func (m *QueryRequest_InputSerialization_ParquetInput) String() string { } func (*QueryRequest_InputSerialization_ParquetInput) ProtoMessage() {} func (*QueryRequest_InputSerialization_ParquetInput) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{56, 1, 2} + return fileDescriptor0, []int{59, 1, 2} } type QueryRequest_OutputSerialization struct { @@ -1629,7 +1702,7 @@ func (m *QueryRequest_OutputSerialization) Reset() { *m = QueryRequest_O func (m *QueryRequest_OutputSerialization) String() string { return proto.CompactTextString(m) } func (*QueryRequest_OutputSerialization) ProtoMessage() {} func (*QueryRequest_OutputSerialization) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{56, 2} + return fileDescriptor0, []int{59, 2} } func (m *QueryRequest_OutputSerialization) GetCsvOutput() *QueryRequest_OutputSerialization_CSVOutput { @@ -1662,7 +1735,7 @@ func (m *QueryRequest_OutputSerialization_CSVOutput) String() string { } func (*QueryRequest_OutputSerialization_CSVOutput) ProtoMessage() {} func (*QueryRequest_OutputSerialization_CSVOutput) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{56, 2, 0} + return fileDescriptor0, []int{59, 2, 0} } func (m *QueryRequest_OutputSerialization_CSVOutput) GetQuoteFields() string { @@ -1712,7 +1785,7 @@ func (m *QueryRequest_OutputSerialization_JSONOutput) String() string { } func (*QueryRequest_OutputSerialization_JSONOutput) ProtoMessage() {} func (*QueryRequest_OutputSerialization_JSONOutput) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{56, 2, 1} + return fileDescriptor0, []int{59, 2, 1} } func (m *QueryRequest_OutputSerialization_JSONOutput) GetRecordDelimiter() string { @@ -1729,7 +1802,7 @@ type QueriedStripe struct { func (m *QueriedStripe) Reset() { *m = QueriedStripe{} } func (m *QueriedStripe) String() string { return proto.CompactTextString(m) } func (*QueriedStripe) ProtoMessage() {} -func (*QueriedStripe) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{57} } +func (*QueriedStripe) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{60} } func (m *QueriedStripe) GetRecords() []byte { if m != nil { @@ -1795,6 +1868,9 @@ func init() { proto.RegisterType((*ReadVolumeFileStatusResponse)(nil), "volume_server_pb.ReadVolumeFileStatusResponse") proto.RegisterType((*DiskStatus)(nil), "volume_server_pb.DiskStatus") proto.RegisterType((*MemStatus)(nil), "volume_server_pb.MemStatus") + proto.RegisterType((*TieredVolume)(nil), "volume_server_pb.TieredVolume") + proto.RegisterType((*VolumeTierCopyDatToRemoteRequest)(nil), "volume_server_pb.VolumeTierCopyDatToRemoteRequest") + proto.RegisterType((*VolumeTierCopyDatToRemoteResponse)(nil), "volume_server_pb.VolumeTierCopyDatToRemoteResponse") proto.RegisterType((*QueryRequest)(nil), "volume_server_pb.QueryRequest") proto.RegisterType((*QueryRequest_Filter)(nil), "volume_server_pb.QueryRequest.Filter") proto.RegisterType((*QueryRequest_InputSerialization)(nil), "volume_server_pb.QueryRequest.InputSerialization") @@ -1847,6 +1923,8 @@ type VolumeServerClient interface { VolumeEcShardsUnmount(ctx context.Context, in *VolumeEcShardsUnmountRequest, opts ...grpc.CallOption) (*VolumeEcShardsUnmountResponse, error) VolumeEcShardRead(ctx context.Context, in *VolumeEcShardReadRequest, opts ...grpc.CallOption) (VolumeServer_VolumeEcShardReadClient, error) VolumeEcBlobDelete(ctx context.Context, in *VolumeEcBlobDeleteRequest, opts ...grpc.CallOption) (*VolumeEcBlobDeleteResponse, error) + // tiered storage + VolumeTierCopyDatToRemote(ctx context.Context, in *VolumeTierCopyDatToRemoteRequest, opts ...grpc.CallOption) (*VolumeTierCopyDatToRemoteResponse, error) // query Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (VolumeServer_QueryClient, error) } @@ -2185,6 +2263,15 @@ func (c *volumeServerClient) VolumeEcBlobDelete(ctx context.Context, in *VolumeE return out, nil } +func (c *volumeServerClient) VolumeTierCopyDatToRemote(ctx context.Context, in *VolumeTierCopyDatToRemoteRequest, opts ...grpc.CallOption) (*VolumeTierCopyDatToRemoteResponse, error) { + out := new(VolumeTierCopyDatToRemoteResponse) + err := grpc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeTierCopyDatToRemote", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *volumeServerClient) Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (VolumeServer_QueryClient, error) { stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[4], c.cc, "/volume_server_pb.VolumeServer/Query", opts...) if err != nil { @@ -2249,6 +2336,8 @@ type VolumeServerServer interface { VolumeEcShardsUnmount(context.Context, *VolumeEcShardsUnmountRequest) (*VolumeEcShardsUnmountResponse, error) VolumeEcShardRead(*VolumeEcShardReadRequest, VolumeServer_VolumeEcShardReadServer) error VolumeEcBlobDelete(context.Context, *VolumeEcBlobDeleteRequest) (*VolumeEcBlobDeleteResponse, error) + // tiered storage + VolumeTierCopyDatToRemote(context.Context, *VolumeTierCopyDatToRemoteRequest) (*VolumeTierCopyDatToRemoteResponse, error) // query Query(*QueryRequest, VolumeServer_QueryServer) error } @@ -2737,6 +2826,24 @@ func _VolumeServer_VolumeEcBlobDelete_Handler(srv interface{}, ctx context.Conte return interceptor(ctx, in, info, handler) } +func _VolumeServer_VolumeTierCopyDatToRemote_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(VolumeTierCopyDatToRemoteRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(VolumeServerServer).VolumeTierCopyDatToRemote(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/volume_server_pb.VolumeServer/VolumeTierCopyDatToRemote", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(VolumeServerServer).VolumeTierCopyDatToRemote(ctx, req.(*VolumeTierCopyDatToRemoteRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _VolumeServer_Query_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(QueryRequest) if err := stream.RecvMsg(m); err != nil { @@ -2850,6 +2957,10 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{ MethodName: "VolumeEcBlobDelete", Handler: _VolumeServer_VolumeEcBlobDelete_Handler, }, + { + MethodName: "VolumeTierCopyDatToRemote", + Handler: _VolumeServer_VolumeTierCopyDatToRemote_Handler, + }, }, Streams: []grpc.StreamDesc{ { @@ -2884,162 +2995,167 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("volume_server.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 2503 bytes of a gzipped FileDescriptorProto + // 2590 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xd4, 0x1a, 0x4d, 0x77, 0x1c, 0x47, - 0x51, 0xeb, 0x5d, 0x49, 0xbb, 0xb5, 0x2b, 0x4b, 0x6e, 0xc9, 0xd2, 0x7a, 0x6c, 0xc9, 0xca, 0xe4, - 0xc3, 0xb2, 0x9d, 0xc8, 0x8e, 0x02, 0x24, 0x24, 0x04, 0xb0, 0x65, 0x1b, 0x4c, 0x62, 0x99, 0x8c, - 0x1c, 0x13, 0x70, 0x1e, 0xf3, 0x5a, 0x33, 0x2d, 0x6b, 0xd0, 0xcc, 0xf4, 0x78, 0xa6, 0x47, 0xd6, - 0xfa, 0xc1, 0x29, 0x5c, 0xf9, 0x01, 0x9c, 0x39, 0x71, 0xe1, 0xca, 0x0f, 0xe0, 0xc2, 0x0f, 0x80, - 0x2b, 0x17, 0xce, 0x1c, 0xb8, 0xf1, 0x1e, 0x17, 0x5e, 0x7f, 0xcc, 0xec, 0x7c, 0x6a, 0x47, 0xb1, - 0xdf, 0xe3, 0xe5, 0xd6, 0x53, 0x5d, 0x1f, 0x5d, 0xd5, 0x55, 0xd5, 0xd5, 0xd5, 0x03, 0x8b, 0x47, - 0xd4, 0x8d, 0x3d, 0x62, 0x46, 0x24, 0x3c, 0x22, 0xe1, 0x66, 0x10, 0x52, 0x46, 0xd1, 0x42, 0x0e, - 0x68, 0x06, 0x7b, 0xfa, 0x0d, 0x40, 0xb7, 0x31, 0xb3, 0x0e, 0xee, 0x10, 0x97, 0x30, 0x62, 0x90, - 0x67, 0x31, 0x89, 0x18, 0xba, 0x00, 0xdd, 0x7d, 0xc7, 0x25, 0xa6, 0x63, 0x47, 0xc3, 0xd6, 0x7a, - 0x7b, 0xa3, 0x67, 0xcc, 0xf2, 0xef, 0xfb, 0x76, 0xa4, 0x3f, 0x84, 0xc5, 0x1c, 0x41, 0x14, 0x50, - 0x3f, 0x22, 0xe8, 0x03, 0x98, 0x0d, 0x49, 0x14, 0xbb, 0x4c, 0x12, 0xf4, 0xb7, 0xd6, 0x36, 0x8b, - 0xb2, 0x36, 0x53, 0x92, 0xd8, 0x65, 0x46, 0x82, 0xae, 0x7f, 0xd5, 0x82, 0x41, 0x76, 0x06, 0xad, - 0xc0, 0xac, 0x12, 0x3e, 0x6c, 0xad, 0xb7, 0x36, 0x7a, 0xc6, 0x8c, 0x94, 0x8d, 0x96, 0x61, 0x26, - 0x62, 0x98, 0xc5, 0xd1, 0xf0, 0xcc, 0x7a, 0x6b, 0x63, 0xda, 0x50, 0x5f, 0x68, 0x09, 0xa6, 0x49, - 0x18, 0xd2, 0x70, 0xd8, 0x16, 0xe8, 0xf2, 0x03, 0x21, 0xe8, 0x44, 0xce, 0x0b, 0x32, 0xec, 0xac, - 0xb7, 0x36, 0xe6, 0x0c, 0x31, 0x46, 0x43, 0x98, 0x3d, 0x22, 0x61, 0xe4, 0x50, 0x7f, 0x38, 0x2d, - 0xc0, 0xc9, 0xa7, 0x3e, 0x0b, 0xd3, 0x77, 0xbd, 0x80, 0x8d, 0xf4, 0xf7, 0x61, 0xf8, 0x18, 0x5b, - 0x71, 0xec, 0x3d, 0x16, 0xcb, 0xdf, 0x3e, 0x20, 0xd6, 0x61, 0x62, 0x96, 0x8b, 0xd0, 0x53, 0x4a, - 0xa9, 0xb5, 0xcd, 0x19, 0x5d, 0x09, 0xb8, 0x6f, 0xeb, 0x3f, 0x84, 0x0b, 0x15, 0x84, 0xca, 0x3c, - 0xaf, 0xc3, 0xdc, 0x53, 0x1c, 0xee, 0xe1, 0xa7, 0xc4, 0x0c, 0x31, 0x73, 0xa8, 0xa0, 0x6e, 0x19, - 0x03, 0x05, 0x34, 0x38, 0x4c, 0x7f, 0x02, 0x5a, 0x8e, 0x03, 0xf5, 0x02, 0x6c, 0xb1, 0x26, 0xc2, - 0xd1, 0x3a, 0xf4, 0x83, 0x90, 0x60, 0xd7, 0xa5, 0x16, 0x66, 0x44, 0xd8, 0xa7, 0x6d, 0x64, 0x41, - 0xfa, 0x2a, 0x5c, 0xac, 0x64, 0x2e, 0x17, 0xa8, 0x7f, 0x50, 0x58, 0x3d, 0xf5, 0x3c, 0xa7, 0x91, - 0x68, 0xfd, 0x52, 0x69, 0xd5, 0x82, 0x52, 0xf1, 0xfd, 0x6e, 0x61, 0xd6, 0x25, 0xd8, 0x8f, 0x83, - 0x46, 0x8c, 0x8b, 0x2b, 0x4e, 0x48, 0x53, 0xce, 0x2b, 0xd2, 0x6d, 0xb6, 0xa9, 0xeb, 0x12, 0x8b, - 0x39, 0xd4, 0x4f, 0xd8, 0xae, 0x01, 0x58, 0x29, 0x50, 0x39, 0x51, 0x06, 0xa2, 0x6b, 0x30, 0x2c, - 0x93, 0x2a, 0xb6, 0xff, 0x68, 0xc1, 0xf9, 0x5b, 0xca, 0x68, 0x52, 0x70, 0xa3, 0x0d, 0xc8, 0x8b, - 0x3c, 0x53, 0x14, 0x59, 0xdc, 0xa0, 0x76, 0x69, 0x83, 0x38, 0x46, 0x48, 0x02, 0xd7, 0xb1, 0xb0, - 0x60, 0xd1, 0x11, 0x2c, 0xb2, 0x20, 0xb4, 0x00, 0x6d, 0xc6, 0x5c, 0xe1, 0xb9, 0x3d, 0x83, 0x0f, - 0xd1, 0x16, 0x2c, 0x7b, 0xc4, 0xa3, 0xe1, 0xc8, 0xf4, 0x70, 0x60, 0x7a, 0xf8, 0xd8, 0xe4, 0x6e, - 0x6e, 0x7a, 0x7b, 0xc3, 0x19, 0xb1, 0x3e, 0x24, 0x67, 0x1f, 0xe0, 0xe0, 0x01, 0x3e, 0xde, 0x75, - 0x5e, 0x90, 0x07, 0x7b, 0xfa, 0x10, 0x96, 0x8b, 0xfa, 0x29, 0xd5, 0xbf, 0x03, 0x2b, 0x12, 0xb2, - 0x3b, 0xf2, 0xad, 0x5d, 0x11, 0x5b, 0x8d, 0x36, 0xea, 0xbf, 0x2d, 0x18, 0x96, 0x09, 0x95, 0xe7, - 0xbf, 0xac, 0xd5, 0x4e, 0x6d, 0x93, 0xcb, 0xd0, 0x67, 0xd8, 0x71, 0x4d, 0xba, 0xbf, 0x1f, 0x11, - 0x26, 0x0c, 0xd1, 0x31, 0x80, 0x83, 0x1e, 0x0a, 0x08, 0xba, 0x0a, 0x0b, 0x96, 0xf4, 0x7e, 0x33, - 0x24, 0x47, 0x8e, 0xc8, 0x06, 0xb3, 0x62, 0x61, 0xf3, 0x56, 0x12, 0x15, 0x12, 0x8c, 0x74, 0x98, - 0x73, 0xec, 0x63, 0x53, 0xa4, 0x23, 0x91, 0x4c, 0xba, 0x82, 0x5b, 0xdf, 0xb1, 0x8f, 0xef, 0x39, - 0x2e, 0xe1, 0x16, 0xd5, 0x1f, 0xc3, 0x25, 0xa9, 0xfc, 0x7d, 0xdf, 0x0a, 0x89, 0x47, 0x7c, 0x86, - 0xdd, 0x6d, 0x1a, 0x8c, 0x1a, 0xb9, 0xcd, 0x05, 0xe8, 0x46, 0x8e, 0x6f, 0x11, 0xd3, 0x97, 0x49, - 0xad, 0x63, 0xcc, 0x8a, 0xef, 0x9d, 0x48, 0xbf, 0x0d, 0xab, 0x35, 0x7c, 0x95, 0x65, 0x5f, 0x83, - 0x81, 0x58, 0x98, 0x45, 0x7d, 0x46, 0x7c, 0x26, 0x78, 0x0f, 0x8c, 0x3e, 0x87, 0x6d, 0x4b, 0x90, - 0xfe, 0x2e, 0x20, 0xc9, 0xe3, 0x01, 0x8d, 0xfd, 0x66, 0xe1, 0x7c, 0x1e, 0x16, 0x73, 0x24, 0xca, - 0x37, 0xde, 0x83, 0x25, 0x09, 0xfe, 0xdc, 0xf7, 0x1a, 0xf3, 0x5a, 0x81, 0xf3, 0x05, 0x22, 0xc5, - 0x6d, 0x2b, 0x11, 0x92, 0x3f, 0x76, 0x4e, 0x64, 0xb6, 0x9c, 0xac, 0x20, 0x7f, 0xf2, 0x88, 0xcc, - 0x25, 0x17, 0x8c, 0xc3, 0x43, 0x83, 0x60, 0x9b, 0xfa, 0xee, 0xa8, 0x71, 0xe6, 0xaa, 0xa0, 0x54, - 0x7c, 0xff, 0xd4, 0x82, 0x73, 0x49, 0x4a, 0x6b, 0xb8, 0x9b, 0xa7, 0x74, 0xe7, 0x76, 0xad, 0x3b, - 0x77, 0xc6, 0xee, 0xbc, 0x01, 0x0b, 0x11, 0x8d, 0x43, 0x8b, 0x98, 0x36, 0x66, 0xd8, 0xf4, 0xa9, - 0x4d, 0x94, 0xb7, 0x9f, 0x95, 0xf0, 0x3b, 0x98, 0xe1, 0x1d, 0x6a, 0x13, 0xfd, 0x07, 0xc9, 0x66, - 0xe7, 0xbc, 0xe4, 0x2a, 0x9c, 0x73, 0x71, 0xc4, 0x4c, 0x1c, 0x04, 0xc4, 0xb7, 0x4d, 0xcc, 0xb8, - 0xab, 0xb5, 0x84, 0xab, 0x9d, 0xe5, 0x13, 0xb7, 0x04, 0xfc, 0x16, 0xdb, 0x89, 0xf4, 0xbf, 0xb5, - 0x60, 0x9e, 0xd3, 0x72, 0xd7, 0x6e, 0xa4, 0xef, 0x02, 0xb4, 0xc9, 0x31, 0x53, 0x8a, 0xf2, 0x21, - 0xba, 0x01, 0x8b, 0x2a, 0x86, 0x1c, 0xea, 0x8f, 0xc3, 0xab, 0x2d, 0xb3, 0xd1, 0x78, 0x2a, 0x8d, - 0xb0, 0xcb, 0xd0, 0x8f, 0x18, 0x0d, 0x92, 0x68, 0xed, 0xc8, 0x68, 0xe5, 0x20, 0x15, 0xad, 0x79, - 0x9b, 0x4e, 0x57, 0xd8, 0x74, 0xe0, 0x44, 0x26, 0xb1, 0x4c, 0xb9, 0x2a, 0x11, 0xef, 0x5d, 0x03, - 0x9c, 0xe8, 0xae, 0x25, 0xad, 0xa1, 0x7f, 0x1b, 0x16, 0xc6, 0x5a, 0x35, 0x8f, 0x9d, 0xaf, 0x5a, - 0x49, 0x3a, 0x7c, 0x84, 0x1d, 0x77, 0x97, 0xf8, 0x36, 0x09, 0x5f, 0x32, 0xa6, 0xd1, 0x4d, 0x58, - 0x72, 0x6c, 0x97, 0x98, 0xcc, 0xf1, 0x08, 0x8d, 0x99, 0x19, 0x11, 0x8b, 0xfa, 0x76, 0x94, 0xd8, - 0x87, 0xcf, 0x3d, 0x92, 0x53, 0xbb, 0x72, 0x46, 0xff, 0x6d, 0x9a, 0x5b, 0xb3, 0xab, 0x18, 0x57, - 0x15, 0x3e, 0x21, 0x9c, 0xe1, 0x01, 0xc1, 0x36, 0x09, 0x95, 0x1a, 0x03, 0x09, 0xfc, 0xb1, 0x80, - 0x71, 0x0b, 0x2b, 0xa4, 0x3d, 0x6a, 0x8f, 0xc4, 0x8a, 0x06, 0x06, 0x48, 0xd0, 0x6d, 0x6a, 0x8f, - 0x44, 0x92, 0x8b, 0x4c, 0xe1, 0x24, 0xd6, 0x41, 0xec, 0x1f, 0x8a, 0xd5, 0x74, 0x8d, 0xbe, 0x13, - 0x7d, 0x8a, 0x23, 0xb6, 0xcd, 0x41, 0xfa, 0x9f, 0x5b, 0x49, 0x94, 0xf1, 0x65, 0x18, 0xc4, 0x22, - 0xce, 0xd1, 0xff, 0xc1, 0x1c, 0x9c, 0x42, 0x45, 0x43, 0xae, 0xba, 0x54, 0x01, 0x83, 0xe4, 0x9c, - 0x3a, 0x8b, 0xc4, 0xcc, 0x38, 0xc8, 0xf3, 0x0b, 0x57, 0x41, 0xfe, 0x65, 0x92, 0x64, 0xef, 0x5a, - 0xbb, 0x07, 0x38, 0xb4, 0xa3, 0x1f, 0x11, 0x9f, 0x84, 0x98, 0xbd, 0x92, 0x43, 0x5f, 0x5f, 0x87, - 0xb5, 0x3a, 0xee, 0x4a, 0xfe, 0x93, 0xe4, 0xf0, 0x48, 0x30, 0x0c, 0xb2, 0x17, 0x3b, 0xae, 0xfd, - 0x4a, 0xc4, 0x7f, 0x52, 0x54, 0x2e, 0x65, 0xae, 0xfc, 0xe7, 0x1a, 0x9c, 0x0b, 0x05, 0x88, 0x99, - 0x11, 0x47, 0x48, 0xeb, 0xfd, 0x39, 0x63, 0x5e, 0x4d, 0x08, 0x42, 0x5e, 0xf7, 0xff, 0x25, 0xf5, - 0x80, 0x84, 0xdb, 0x2b, 0x4b, 0x8b, 0x17, 0xa1, 0x37, 0x16, 0xdf, 0x16, 0xe2, 0xbb, 0x91, 0x92, - 0xcb, 0xbd, 0xd3, 0xa2, 0xc1, 0xc8, 0x24, 0x96, 0x3c, 0x87, 0xc5, 0x56, 0x77, 0x8d, 0x3e, 0x07, - 0xde, 0xb5, 0xc4, 0x31, 0x7c, 0x8a, 0x1c, 0x99, 0x7a, 0x43, 0x5e, 0x09, 0xb5, 0x1b, 0xcf, 0xe1, - 0x62, 0x7e, 0xb6, 0xf9, 0xf1, 0xf4, 0x52, 0x4a, 0xea, 0x6b, 0x45, 0x37, 0x28, 0x9c, 0x71, 0x47, - 0xc5, 0x65, 0x37, 0x3e, 0xcf, 0x5f, 0x6e, 0x5d, 0xab, 0x45, 0x83, 0xe4, 0x8b, 0x82, 0x2f, 0x8a, - 0xcb, 0x3e, 0x45, 0x71, 0x70, 0xb2, 0xe0, 0xcb, 0x45, 0xd7, 0x2d, 0x56, 0x10, 0xbf, 0x4f, 0xf3, - 0xa2, 0xc2, 0xe0, 0xe7, 0x77, 0xe3, 0x7c, 0xa4, 0xe4, 0x0a, 0x73, 0xcc, 0x19, 0xb3, 0x4a, 0x2c, - 0xbf, 0x60, 0xaa, 0x73, 0x48, 0xd6, 0xe7, 0xea, 0x2b, 0x77, 0x95, 0x6c, 0xab, 0xab, 0x64, 0x72, - 0x45, 0x3e, 0x24, 0x23, 0xe1, 0x6b, 0x1d, 0x79, 0x45, 0xfe, 0x84, 0x8c, 0xf4, 0x9d, 0x42, 0xa4, - 0xc8, 0xa5, 0xa9, 0x98, 0x43, 0xd0, 0xe1, 0x4e, 0xaa, 0x52, 0xb5, 0x18, 0xa3, 0x55, 0x00, 0x27, - 0x32, 0x6d, 0xb1, 0xe7, 0x72, 0x51, 0x5d, 0xa3, 0xe7, 0x28, 0x27, 0xb0, 0xf5, 0xdf, 0x65, 0x42, - 0xef, 0xb6, 0x4b, 0xf7, 0x5e, 0xa1, 0x57, 0x66, 0xb5, 0x68, 0xe7, 0xb4, 0xc8, 0xde, 0x95, 0x3b, - 0xf9, 0xbb, 0x72, 0x26, 0x88, 0xb2, 0xcb, 0x51, 0x3b, 0xf3, 0x21, 0x5c, 0xe4, 0x0a, 0x4b, 0x0c, - 0x51, 0x25, 0x37, 0xbf, 0x49, 0xfc, 0xeb, 0x0c, 0x5c, 0xaa, 0x26, 0x6e, 0x72, 0x9b, 0xf8, 0x08, - 0xb4, 0xb4, 0x5a, 0xe7, 0x47, 0x4a, 0xc4, 0xb0, 0x17, 0xa4, 0x87, 0x8a, 0x3c, 0x7b, 0x56, 0x54, - 0xe9, 0xfe, 0x28, 0x99, 0x4f, 0x4e, 0x96, 0x52, 0xa9, 0xdf, 0x2e, 0x95, 0xfa, 0x5c, 0x80, 0x8d, - 0x59, 0x9d, 0x00, 0x59, 0xbb, 0xac, 0xd8, 0x98, 0xd5, 0x09, 0x48, 0x89, 0x85, 0x00, 0xe9, 0x35, - 0x7d, 0x85, 0x2f, 0x04, 0xac, 0x02, 0xa8, 0xb2, 0x24, 0xf6, 0x93, 0xab, 0x4b, 0x4f, 0x16, 0x25, - 0xb1, 0x5f, 0x5b, 0x5d, 0xcd, 0xd6, 0x56, 0x57, 0xf9, 0xed, 0xef, 0x96, 0x4e, 0x88, 0x2f, 0x00, - 0xee, 0x38, 0xd1, 0xa1, 0x34, 0x32, 0x2f, 0xe7, 0x6c, 0x27, 0x54, 0xf7, 0x65, 0x3e, 0xe4, 0x10, - 0xec, 0xba, 0xca, 0x74, 0x7c, 0xc8, 0xdd, 0x37, 0x8e, 0x88, 0xad, 0xac, 0x23, 0xc6, 0x1c, 0xb6, - 0x1f, 0x12, 0xa2, 0x0c, 0x20, 0xc6, 0xfa, 0x1f, 0x5a, 0xd0, 0x7b, 0x40, 0x3c, 0xc5, 0x79, 0x0d, - 0xe0, 0x29, 0x0d, 0x69, 0xcc, 0x1c, 0x9f, 0xc8, 0xea, 0x73, 0xda, 0xc8, 0x40, 0xbe, 0xbe, 0x1c, - 0x11, 0x9a, 0xc4, 0xdd, 0x57, 0xc6, 0x14, 0x63, 0x0e, 0x3b, 0x20, 0x38, 0x50, 0xf6, 0x13, 0x63, - 0xb4, 0x04, 0xd3, 0x11, 0xc3, 0xd6, 0xa1, 0x30, 0x56, 0xc7, 0x90, 0x1f, 0xfa, 0x7f, 0x06, 0x30, - 0xf8, 0x2c, 0x26, 0xe1, 0x28, 0xd3, 0x39, 0x88, 0x88, 0xb2, 0x4e, 0xd2, 0xfa, 0xca, 0x40, 0xf8, - 0x26, 0xee, 0x87, 0xd4, 0x33, 0xd3, 0xee, 0xd8, 0x19, 0x81, 0xd2, 0xe7, 0xc0, 0x7b, 0xb2, 0x43, - 0x86, 0x3e, 0x86, 0x99, 0x7d, 0xc7, 0x65, 0x44, 0xf6, 0xa3, 0xfa, 0x5b, 0x6f, 0x96, 0x3b, 0x61, - 0x59, 0x99, 0x9b, 0xf7, 0x04, 0xb2, 0xa1, 0x88, 0xd0, 0x1e, 0x2c, 0x3a, 0x7e, 0x20, 0xaa, 0xa1, - 0xd0, 0xc1, 0xae, 0xf3, 0x62, 0x7c, 0xf7, 0xed, 0x6f, 0xbd, 0x3b, 0x81, 0xd7, 0x7d, 0x4e, 0xb9, - 0x9b, 0x25, 0x34, 0x90, 0x53, 0x82, 0x21, 0x02, 0x4b, 0x34, 0x66, 0x65, 0x21, 0xd3, 0x42, 0xc8, - 0xd6, 0x04, 0x21, 0x0f, 0x05, 0x69, 0x5e, 0xca, 0x22, 0x2d, 0x03, 0xb5, 0x1d, 0x98, 0x91, 0xca, - 0x71, 0xf3, 0xef, 0x3b, 0xc4, 0x4d, 0x3a, 0x7a, 0xf2, 0x83, 0xa7, 0x18, 0x1a, 0x90, 0x10, 0xfb, - 0xb6, 0x4a, 0x4d, 0xc9, 0x27, 0xc7, 0x3f, 0xc2, 0x6e, 0x4c, 0x92, 0x96, 0x9e, 0xf8, 0xd0, 0xfe, - 0x3e, 0x0d, 0xa8, 0xac, 0x61, 0x72, 0xa1, 0x0f, 0x49, 0xc4, 0x9d, 0xde, 0x64, 0xa3, 0x80, 0x28, - 0x39, 0xf3, 0x19, 0xf8, 0xa3, 0x51, 0x40, 0xd0, 0xcf, 0xa0, 0x67, 0x45, 0x47, 0xa6, 0x30, 0x89, - 0x90, 0xd9, 0xdf, 0xfa, 0xf0, 0xd4, 0x26, 0xdd, 0xdc, 0xde, 0x7d, 0x2c, 0xa0, 0x46, 0xd7, 0x8a, - 0x8e, 0xc4, 0x08, 0xfd, 0x02, 0xe0, 0x57, 0x11, 0xf5, 0x15, 0x67, 0xb9, 0xf1, 0x1f, 0x9d, 0x9e, - 0xf3, 0x4f, 0x76, 0x1f, 0xee, 0x48, 0xd6, 0x3d, 0xce, 0x4e, 0xf2, 0xb6, 0x60, 0x2e, 0xc0, 0xe1, - 0xb3, 0x98, 0x30, 0xc5, 0x5e, 0xfa, 0xc2, 0xf7, 0x4f, 0xcf, 0xfe, 0xa7, 0x92, 0x8d, 0x94, 0x30, - 0x08, 0x32, 0x5f, 0xda, 0x5f, 0xcf, 0x40, 0x37, 0xd1, 0x8b, 0x17, 0x54, 0xc2, 0xc3, 0xe5, 0xb5, - 0xc2, 0x74, 0xfc, 0x7d, 0xaa, 0x2c, 0x7a, 0x96, 0xc3, 0xe5, 0xcd, 0xe2, 0xbe, 0xbf, 0x4f, 0xb9, - 0xed, 0x43, 0x62, 0xd1, 0xd0, 0xe6, 0xc7, 0x97, 0xe3, 0x39, 0xdc, 0xed, 0xe5, 0x5e, 0xce, 0x4b, - 0xf8, 0x9d, 0x04, 0x8c, 0xae, 0xc0, 0xbc, 0xd8, 0xf6, 0x0c, 0x66, 0x3b, 0xe1, 0x49, 0xdc, 0x0c, - 0xe2, 0x55, 0x58, 0x78, 0x16, 0x53, 0x46, 0x4c, 0xeb, 0x00, 0x87, 0xd8, 0x62, 0x34, 0x2d, 0xf0, - 0xe7, 0x05, 0x7c, 0x3b, 0x05, 0xa3, 0x6f, 0xc1, 0xb2, 0x44, 0x25, 0x91, 0x85, 0x83, 0x94, 0x82, - 0x84, 0xaa, 0xfe, 0x5b, 0x12, 0xb3, 0x77, 0xc5, 0xe4, 0x76, 0x32, 0x87, 0x34, 0xe8, 0x5a, 0xd4, - 0xf3, 0x88, 0xcf, 0x22, 0x91, 0x24, 0x7a, 0x46, 0xfa, 0x8d, 0x6e, 0xc1, 0x2a, 0x76, 0x5d, 0xfa, - 0xdc, 0x14, 0x94, 0xb6, 0x59, 0xd2, 0x6e, 0x56, 0x1c, 0xcf, 0x9a, 0x40, 0xfa, 0x4c, 0xe0, 0x18, - 0x79, 0x45, 0xb5, 0xcb, 0xd0, 0x4b, 0xf7, 0x91, 0x27, 0xa3, 0x8c, 0x43, 0x8a, 0xb1, 0x76, 0x16, - 0x06, 0xd9, 0x9d, 0xd0, 0xfe, 0xdd, 0x86, 0xc5, 0x8a, 0xa0, 0x42, 0x4f, 0x00, 0xb8, 0xb7, 0xca, - 0xd0, 0x52, 0xee, 0xfa, 0xbd, 0xd3, 0x07, 0x27, 0xf7, 0x57, 0x09, 0x36, 0xb8, 0xf7, 0xcb, 0x21, - 0xfa, 0x25, 0xf4, 0x85, 0xc7, 0x2a, 0xee, 0xd2, 0x65, 0x3f, 0xfe, 0x1a, 0xdc, 0xb9, 0xae, 0x8a, - 0xbd, 0x88, 0x01, 0x39, 0xd6, 0xfe, 0xd9, 0x82, 0x5e, 0x2a, 0x98, 0x5f, 0xb8, 0xe5, 0x46, 0x89, - 0xbd, 0x8e, 0x94, 0x39, 0xfa, 0x02, 0x76, 0x4f, 0x80, 0xbe, 0x91, 0xae, 0xa4, 0xbd, 0x0f, 0x30, - 0xd6, 0xbf, 0x52, 0x85, 0x56, 0xa5, 0x0a, 0xfa, 0x55, 0x98, 0xe3, 0x96, 0x75, 0x88, 0xbd, 0xcb, - 0x42, 0x27, 0x10, 0x6f, 0x13, 0x12, 0x27, 0x52, 0xb5, 0x61, 0xf2, 0xb9, 0xf5, 0xc7, 0x15, 0x18, - 0x64, 0xef, 0xb4, 0xe8, 0x4b, 0xe8, 0x67, 0xde, 0x60, 0xd0, 0x1b, 0xe5, 0x4d, 0x2b, 0xbf, 0xe9, - 0x68, 0x6f, 0x4e, 0xc0, 0x52, 0xe5, 0xdb, 0x14, 0xf2, 0xe1, 0x5c, 0xe9, 0x21, 0x03, 0x5d, 0x2b, - 0x53, 0xd7, 0x3d, 0x93, 0x68, 0xd7, 0x1b, 0xe1, 0xa6, 0xf2, 0x18, 0x2c, 0x56, 0xbc, 0x4c, 0xa0, - 0xb7, 0x27, 0x70, 0xc9, 0xbd, 0x8e, 0x68, 0xef, 0x34, 0xc4, 0x4e, 0xa5, 0x3e, 0x03, 0x54, 0x7e, - 0xb6, 0x40, 0xd7, 0x27, 0xb2, 0x19, 0x3f, 0x8b, 0x68, 0x6f, 0x37, 0x43, 0xae, 0x55, 0x54, 0x3e, - 0x68, 0x4c, 0x54, 0x34, 0xf7, 0x64, 0x32, 0x51, 0xd1, 0xc2, 0x2b, 0xc9, 0x14, 0x3a, 0x84, 0x85, - 0xe2, 0x63, 0x07, 0xba, 0x5a, 0xf7, 0x38, 0x57, 0x7a, 0x4b, 0xd1, 0xae, 0x35, 0x41, 0x4d, 0x85, - 0x11, 0x38, 0x9b, 0x7f, 0x5c, 0x40, 0x57, 0xca, 0xf4, 0x95, 0xcf, 0x2b, 0xda, 0xc6, 0x64, 0xc4, - 0xac, 0x4e, 0xc5, 0x07, 0x87, 0x2a, 0x9d, 0x6a, 0x5e, 0x33, 0xaa, 0x74, 0xaa, 0x7b, 0xbf, 0xd0, - 0xa7, 0xd0, 0xaf, 0x93, 0x2e, 0x76, 0xa1, 0x11, 0x8f, 0x36, 0xeb, 0xd8, 0x54, 0xbf, 0x04, 0x68, - 0x37, 0x1a, 0xe3, 0x27, 0xb2, 0x6f, 0xb6, 0x78, 0xac, 0x67, 0xfa, 0xf1, 0x55, 0xb1, 0x5e, 0xee, - 0xf0, 0x57, 0xc5, 0x7a, 0x55, 0x53, 0x7f, 0x0a, 0xed, 0xc1, 0x5c, 0xae, 0x43, 0x8f, 0xde, 0xaa, - 0xa3, 0xcc, 0x5f, 0xed, 0xb5, 0x2b, 0x13, 0xf1, 0x52, 0x19, 0x66, 0x92, 0xbd, 0x54, 0xba, 0xaa, - 0x5d, 0x5c, 0x3e, 0x5f, 0xbd, 0x35, 0x09, 0x2d, 0x17, 0xca, 0xa5, 0x3e, 0x7e, 0x65, 0x28, 0xd7, - 0xbd, 0x13, 0x54, 0x86, 0x72, 0xfd, 0xd3, 0xc0, 0x14, 0xfa, 0x39, 0xc0, 0xb8, 0xd7, 0x8e, 0x5e, - 0xaf, 0xa3, 0xce, 0xee, 0xfe, 0x1b, 0x27, 0x23, 0xa5, 0xac, 0x9f, 0xc3, 0x52, 0xd5, 0x15, 0x18, - 0x55, 0x04, 0xfe, 0x09, 0xf7, 0x6c, 0x6d, 0xb3, 0x29, 0x7a, 0x2a, 0xf8, 0x73, 0xe8, 0x26, 0x7d, - 0x72, 0xf4, 0x5a, 0x99, 0xba, 0xf0, 0x32, 0xa0, 0xe9, 0x27, 0xa1, 0x64, 0x1c, 0xd8, 0x4b, 0x62, - 0x75, 0xdc, 0xc0, 0xae, 0x8f, 0xd5, 0x52, 0xab, 0xbd, 0x3e, 0x56, 0xcb, 0xfd, 0x70, 0x21, 0x2e, - 0x75, 0x86, 0x6c, 0xbf, 0xb7, 0xde, 0x19, 0x2a, 0xda, 0xd9, 0xf5, 0xce, 0x50, 0xd9, 0x42, 0x9e, - 0x42, 0xbf, 0x81, 0xe5, 0xea, 0x36, 0x2f, 0xaa, 0x8d, 0xf8, 0x9a, 0x76, 0xb3, 0x76, 0xb3, 0x39, - 0x41, 0x2a, 0xfe, 0x45, 0x92, 0x9f, 0x0a, 0x6d, 0xde, 0xfa, 0xfc, 0x54, 0xdd, 0x6c, 0xd6, 0x6e, - 0x34, 0xc6, 0x2f, 0x87, 0x5e, 0xb6, 0x9f, 0x5a, 0x6f, 0xed, 0x8a, 0xd6, 0x71, 0xbd, 0xb5, 0x2b, - 0x5b, 0xb4, 0x22, 0x3e, 0xaa, 0x7a, 0xa5, 0x55, 0xf1, 0x71, 0x42, 0x33, 0x57, 0xdb, 0x6c, 0x8a, - 0x9e, 0x3b, 0xbe, 0xcb, 0xcd, 0x50, 0x34, 0x71, 0xfd, 0xb9, 0xcc, 0xfc, 0x4e, 0x43, 0xec, 0xfa, - 0xdd, 0x4d, 0x32, 0xf5, 0x44, 0x05, 0x0a, 0x19, 0xfb, 0x46, 0x63, 0xfc, 0x54, 0x76, 0x90, 0xbc, - 0x80, 0x66, 0x1a, 0x99, 0xe8, 0xda, 0x04, 0x3e, 0x99, 0x46, 0xac, 0x76, 0xbd, 0x11, 0x6e, 0x55, - 0xf4, 0x66, 0x5b, 0x8b, 0x27, 0xf9, 0x53, 0xa9, 0x1f, 0x7a, 0x92, 0x3f, 0x55, 0x74, 0x2b, 0xa7, - 0xd0, 0xa7, 0x30, 0x2d, 0xae, 0x38, 0x68, 0xed, 0xe4, 0xbb, 0x8f, 0x76, 0xb9, 0x7a, 0x3e, 0xad, - 0xe0, 0xb9, 0x02, 0x7b, 0x33, 0xe2, 0x47, 0xab, 0xf7, 0xfe, 0x17, 0x00, 0x00, 0xff, 0xff, 0x02, - 0xd4, 0xce, 0xae, 0x7f, 0x25, 0x00, 0x00, + 0x51, 0xeb, 0xd5, 0xc7, 0x6e, 0xed, 0xca, 0x92, 0x5b, 0x8a, 0xb4, 0x19, 0x47, 0xb2, 0x3c, 0x4e, + 0x62, 0xd9, 0x4e, 0x64, 0x47, 0x06, 0x12, 0x12, 0x02, 0xd8, 0xb2, 0x0d, 0x26, 0xb1, 0x4c, 0x46, + 0x8a, 0x09, 0x38, 0x8f, 0x79, 0xbd, 0x33, 0x2d, 0x6b, 0xd0, 0xcc, 0xf4, 0x78, 0xa6, 0x47, 0xf6, + 0xfa, 0xc1, 0x29, 0x3c, 0x6e, 0xfc, 0x00, 0xce, 0xdc, 0x39, 0x70, 0xe1, 0x07, 0x70, 0xe1, 0x07, + 0xc0, 0x95, 0x0b, 0x67, 0x0e, 0xdc, 0x78, 0x8f, 0x0b, 0xaf, 0x3f, 0x66, 0x76, 0x3e, 0xb5, 0xa3, + 0xd8, 0xef, 0xf1, 0xb8, 0xcd, 0x56, 0xd7, 0x47, 0x57, 0x75, 0x55, 0x75, 0x75, 0xd5, 0xc2, 0xd2, + 0x31, 0x75, 0x63, 0x8f, 0x98, 0x11, 0x09, 0x8f, 0x49, 0xb8, 0x15, 0x84, 0x94, 0x51, 0xb4, 0x98, + 0x03, 0x9a, 0xc1, 0x50, 0xbf, 0x0e, 0xe8, 0x36, 0x66, 0xd6, 0xe1, 0x1d, 0xe2, 0x12, 0x46, 0x0c, + 0xf2, 0x34, 0x26, 0x11, 0x43, 0xaf, 0x43, 0xe7, 0xc0, 0x71, 0x89, 0xe9, 0xd8, 0xd1, 0xa0, 0xb5, + 0xd1, 0xde, 0xec, 0x1a, 0x73, 0xfc, 0xf7, 0x7d, 0x3b, 0xd2, 0x1f, 0xc2, 0x52, 0x8e, 0x20, 0x0a, + 0xa8, 0x1f, 0x11, 0xf4, 0x01, 0xcc, 0x85, 0x24, 0x8a, 0x5d, 0x26, 0x09, 0x7a, 0xdb, 0xeb, 0x5b, + 0x45, 0x59, 0x5b, 0x29, 0x49, 0xec, 0x32, 0x23, 0x41, 0xd7, 0xbf, 0x6a, 0x41, 0x3f, 0xbb, 0x82, + 0x56, 0x61, 0x4e, 0x09, 0x1f, 0xb4, 0x36, 0x5a, 0x9b, 0x5d, 0x63, 0x56, 0xca, 0x46, 0x2b, 0x30, + 0x1b, 0x31, 0xcc, 0xe2, 0x68, 0x70, 0x66, 0xa3, 0xb5, 0x39, 0x63, 0xa8, 0x5f, 0x68, 0x19, 0x66, + 0x48, 0x18, 0xd2, 0x70, 0xd0, 0x16, 0xe8, 0xf2, 0x07, 0x42, 0x30, 0x1d, 0x39, 0x2f, 0xc8, 0x60, + 0x7a, 0xa3, 0xb5, 0x39, 0x6f, 0x88, 0x6f, 0x34, 0x80, 0xb9, 0x63, 0x12, 0x46, 0x0e, 0xf5, 0x07, + 0x33, 0x02, 0x9c, 0xfc, 0xd4, 0xe7, 0x60, 0xe6, 0xae, 0x17, 0xb0, 0x91, 0xfe, 0x3e, 0x0c, 0x1e, + 0x61, 0x2b, 0x8e, 0xbd, 0x47, 0x62, 0xfb, 0x3b, 0x87, 0xc4, 0x3a, 0x4a, 0xcc, 0x72, 0x1e, 0xba, + 0x4a, 0x29, 0xb5, 0xb7, 0x79, 0xa3, 0x23, 0x01, 0xf7, 0x6d, 0xfd, 0xfb, 0xf0, 0x7a, 0x05, 0xa1, + 0x32, 0xcf, 0x25, 0x98, 0x7f, 0x82, 0xc3, 0x21, 0x7e, 0x42, 0xcc, 0x10, 0x33, 0x87, 0x0a, 0xea, + 0x96, 0xd1, 0x57, 0x40, 0x83, 0xc3, 0xf4, 0xc7, 0xa0, 0xe5, 0x38, 0x50, 0x2f, 0xc0, 0x16, 0x6b, + 0x22, 0x1c, 0x6d, 0x40, 0x2f, 0x08, 0x09, 0x76, 0x5d, 0x6a, 0x61, 0x46, 0x84, 0x7d, 0xda, 0x46, + 0x16, 0xa4, 0xaf, 0xc1, 0xf9, 0x4a, 0xe6, 0x72, 0x83, 0xfa, 0x07, 0x85, 0xdd, 0x53, 0xcf, 0x73, + 0x1a, 0x89, 0xd6, 0xdf, 0x28, 0xed, 0x5a, 0x50, 0x2a, 0xbe, 0xdf, 0x2e, 0xac, 0xba, 0x04, 0xfb, + 0x71, 0xd0, 0x88, 0x71, 0x71, 0xc7, 0x09, 0x69, 0xca, 0x79, 0x55, 0xba, 0xcd, 0x0e, 0x75, 0x5d, + 0x62, 0x31, 0x87, 0xfa, 0x09, 0xdb, 0x75, 0x00, 0x2b, 0x05, 0x2a, 0x27, 0xca, 0x40, 0x74, 0x0d, + 0x06, 0x65, 0x52, 0xc5, 0xf6, 0xef, 0x2d, 0x78, 0xed, 0x96, 0x32, 0x9a, 0x14, 0xdc, 0xe8, 0x00, + 0xf2, 0x22, 0xcf, 0x14, 0x45, 0x16, 0x0f, 0xa8, 0x5d, 0x3a, 0x20, 0x8e, 0x11, 0x92, 0xc0, 0x75, + 0x2c, 0x2c, 0x58, 0x4c, 0x0b, 0x16, 0x59, 0x10, 0x5a, 0x84, 0x36, 0x63, 0xae, 0xf0, 0xdc, 0xae, + 0xc1, 0x3f, 0xd1, 0x36, 0xac, 0x78, 0xc4, 0xa3, 0xe1, 0xc8, 0xf4, 0x70, 0x60, 0x7a, 0xf8, 0xb9, + 0xc9, 0xdd, 0xdc, 0xf4, 0x86, 0x83, 0x59, 0xb1, 0x3f, 0x24, 0x57, 0x1f, 0xe0, 0xe0, 0x01, 0x7e, + 0xbe, 0xe7, 0xbc, 0x20, 0x0f, 0x86, 0xfa, 0x00, 0x56, 0x8a, 0xfa, 0x29, 0xd5, 0xbf, 0x05, 0xab, + 0x12, 0xb2, 0x37, 0xf2, 0xad, 0x3d, 0x11, 0x5b, 0x8d, 0x0e, 0xea, 0x3f, 0x2d, 0x18, 0x94, 0x09, + 0x95, 0xe7, 0xbf, 0xac, 0xd5, 0x4e, 0x6d, 0x93, 0x0b, 0xd0, 0x63, 0xd8, 0x71, 0x4d, 0x7a, 0x70, + 0x10, 0x11, 0x26, 0x0c, 0x31, 0x6d, 0x00, 0x07, 0x3d, 0x14, 0x10, 0x74, 0x05, 0x16, 0x2d, 0xe9, + 0xfd, 0x66, 0x48, 0x8e, 0x1d, 0x91, 0x0d, 0xe6, 0xc4, 0xc6, 0x16, 0xac, 0x24, 0x2a, 0x24, 0x18, + 0xe9, 0x30, 0xef, 0xd8, 0xcf, 0x4d, 0x91, 0x8e, 0x44, 0x32, 0xe9, 0x08, 0x6e, 0x3d, 0xc7, 0x7e, + 0x7e, 0xcf, 0x71, 0x09, 0xb7, 0xa8, 0xfe, 0x08, 0xde, 0x90, 0xca, 0xdf, 0xf7, 0xad, 0x90, 0x78, + 0xc4, 0x67, 0xd8, 0xdd, 0xa1, 0xc1, 0xa8, 0x91, 0xdb, 0xbc, 0x0e, 0x9d, 0xc8, 0xf1, 0x2d, 0x62, + 0xfa, 0x32, 0xa9, 0x4d, 0x1b, 0x73, 0xe2, 0xf7, 0x6e, 0xa4, 0xdf, 0x86, 0xb5, 0x1a, 0xbe, 0xca, + 0xb2, 0x17, 0xa1, 0x2f, 0x36, 0x66, 0x51, 0x9f, 0x11, 0x9f, 0x09, 0xde, 0x7d, 0xa3, 0xc7, 0x61, + 0x3b, 0x12, 0xa4, 0xbf, 0x07, 0x48, 0xf2, 0x78, 0x40, 0x63, 0xbf, 0x59, 0x38, 0xbf, 0x06, 0x4b, + 0x39, 0x12, 0xe5, 0x1b, 0x37, 0x61, 0x59, 0x82, 0x3f, 0xf7, 0xbd, 0xc6, 0xbc, 0x56, 0xe1, 0xb5, + 0x02, 0x91, 0xe2, 0xb6, 0x9d, 0x08, 0xc9, 0x5f, 0x3b, 0x27, 0x32, 0x5b, 0x49, 0x76, 0x90, 0xbf, + 0x79, 0x44, 0xe6, 0x92, 0x1b, 0xc6, 0xe1, 0x91, 0x41, 0xb0, 0x4d, 0x7d, 0x77, 0xd4, 0x38, 0x73, + 0x55, 0x50, 0x2a, 0xbe, 0x7f, 0x68, 0xc1, 0xb9, 0x24, 0xa5, 0x35, 0x3c, 0xcd, 0x53, 0xba, 0x73, + 0xbb, 0xd6, 0x9d, 0xa7, 0xc7, 0xee, 0xbc, 0x09, 0x8b, 0x11, 0x8d, 0x43, 0x8b, 0x98, 0x36, 0x66, + 0xd8, 0xf4, 0xa9, 0x4d, 0x94, 0xb7, 0x9f, 0x95, 0xf0, 0x3b, 0x98, 0xe1, 0x5d, 0x6a, 0x13, 0xfd, + 0x7b, 0xc9, 0x61, 0xe7, 0xbc, 0xe4, 0x0a, 0x9c, 0x73, 0x71, 0xc4, 0x4c, 0x1c, 0x04, 0xc4, 0xb7, + 0x4d, 0xcc, 0xb8, 0xab, 0xb5, 0x84, 0xab, 0x9d, 0xe5, 0x0b, 0xb7, 0x04, 0xfc, 0x16, 0xdb, 0x8d, + 0xf4, 0xbf, 0xb6, 0x60, 0x81, 0xd3, 0x72, 0xd7, 0x6e, 0xa4, 0xef, 0x22, 0xb4, 0xc9, 0x73, 0xa6, + 0x14, 0xe5, 0x9f, 0xe8, 0x3a, 0x2c, 0xa9, 0x18, 0x72, 0xa8, 0x3f, 0x0e, 0xaf, 0xb6, 0xcc, 0x46, + 0xe3, 0xa5, 0x34, 0xc2, 0x2e, 0x40, 0x2f, 0x62, 0x34, 0x48, 0xa2, 0x75, 0x5a, 0x46, 0x2b, 0x07, + 0xa9, 0x68, 0xcd, 0xdb, 0x74, 0xa6, 0xc2, 0xa6, 0x7d, 0x27, 0x32, 0x89, 0x65, 0xca, 0x5d, 0x89, + 0x78, 0xef, 0x18, 0xe0, 0x44, 0x77, 0x2d, 0x69, 0x0d, 0xfd, 0x9b, 0xb0, 0x38, 0xd6, 0xaa, 0x79, + 0xec, 0x7c, 0xd5, 0x4a, 0xd2, 0xe1, 0x3e, 0x76, 0xdc, 0x3d, 0xe2, 0xdb, 0x24, 0x7c, 0xc9, 0x98, + 0x46, 0x37, 0x60, 0xd9, 0xb1, 0x5d, 0x62, 0x32, 0xc7, 0x23, 0x34, 0x66, 0x66, 0x44, 0x2c, 0xea, + 0xdb, 0x51, 0x62, 0x1f, 0xbe, 0xb6, 0x2f, 0x97, 0xf6, 0xe4, 0x8a, 0xfe, 0xeb, 0x34, 0xb7, 0x66, + 0x77, 0x31, 0xae, 0x2a, 0x7c, 0x42, 0x38, 0xc3, 0x43, 0x82, 0x6d, 0x12, 0x2a, 0x35, 0xfa, 0x12, + 0xf8, 0x43, 0x01, 0xe3, 0x16, 0x56, 0x48, 0x43, 0x6a, 0x8f, 0xc4, 0x8e, 0xfa, 0x06, 0x48, 0xd0, + 0x6d, 0x6a, 0x8f, 0x44, 0x92, 0x8b, 0x4c, 0xe1, 0x24, 0xd6, 0x61, 0xec, 0x1f, 0x89, 0xdd, 0x74, + 0x8c, 0x9e, 0x13, 0x7d, 0x8a, 0x23, 0xb6, 0xc3, 0x41, 0xfa, 0x9f, 0x5a, 0x49, 0x94, 0xf1, 0x6d, + 0x18, 0xc4, 0x22, 0xce, 0xf1, 0xff, 0xc0, 0x1c, 0x9c, 0x42, 0x45, 0x43, 0xae, 0xba, 0x54, 0x01, + 0x83, 0xe4, 0x9a, 0xba, 0x8b, 0xc4, 0xca, 0x38, 0xc8, 0xf3, 0x1b, 0x57, 0x41, 0xfe, 0x65, 0x92, + 0x64, 0xef, 0x5a, 0x7b, 0x87, 0x38, 0xb4, 0xa3, 0x1f, 0x10, 0x9f, 0x84, 0x98, 0xbd, 0x92, 0x4b, + 0x5f, 0xdf, 0x80, 0xf5, 0x3a, 0xee, 0x4a, 0xfe, 0xe3, 0xe4, 0xf2, 0x48, 0x30, 0x0c, 0x32, 0x8c, + 0x1d, 0xd7, 0x7e, 0x25, 0xe2, 0x3f, 0x29, 0x2a, 0x97, 0x32, 0x57, 0xfe, 0x73, 0x15, 0xce, 0x85, + 0x02, 0xc4, 0xcc, 0x88, 0x23, 0xa4, 0xf5, 0xfe, 0xbc, 0xb1, 0xa0, 0x16, 0x04, 0x21, 0xaf, 0xfb, + 0xff, 0x9c, 0x7a, 0x40, 0xc2, 0xed, 0x95, 0xa5, 0xc5, 0xf3, 0xd0, 0x1d, 0x8b, 0x6f, 0x0b, 0xf1, + 0x9d, 0x48, 0xc9, 0xe5, 0xde, 0x69, 0xd1, 0x60, 0x64, 0x12, 0x4b, 0xde, 0xc3, 0xe2, 0xa8, 0x3b, + 0x46, 0x8f, 0x03, 0xef, 0x5a, 0xe2, 0x1a, 0x3e, 0x45, 0x8e, 0x4c, 0xbd, 0x21, 0xaf, 0x84, 0x3a, + 0x8d, 0x67, 0x70, 0x3e, 0xbf, 0xda, 0xfc, 0x7a, 0x7a, 0x29, 0x25, 0xf5, 0xf5, 0xa2, 0x1b, 0x14, + 0xee, 0xb8, 0xe3, 0xe2, 0xb6, 0x1b, 0xdf, 0xe7, 0x2f, 0xb7, 0xaf, 0xb5, 0xa2, 0x41, 0xf2, 0x45, + 0xc1, 0x17, 0xc5, 0x6d, 0x9f, 0xa2, 0x38, 0x38, 0x59, 0xf0, 0x85, 0xa2, 0xeb, 0x16, 0x2b, 0x88, + 0xdf, 0xa5, 0x79, 0x51, 0x61, 0xf0, 0xfb, 0xbb, 0x71, 0x3e, 0x52, 0x72, 0x85, 0x39, 0xe6, 0x8d, + 0x39, 0x25, 0x96, 0x3f, 0x30, 0xd5, 0x3d, 0x24, 0xeb, 0x73, 0xf5, 0x2b, 0xf7, 0x94, 0x6c, 0xab, + 0xa7, 0x64, 0xf2, 0x44, 0x3e, 0x22, 0x23, 0xe1, 0x6b, 0xd3, 0xf2, 0x89, 0xfc, 0x09, 0x19, 0xe9, + 0xbb, 0x85, 0x48, 0x91, 0x5b, 0x53, 0x31, 0x87, 0x60, 0x9a, 0x3b, 0xa9, 0x4a, 0xd5, 0xe2, 0x1b, + 0xad, 0x01, 0x38, 0x91, 0x69, 0x8b, 0x33, 0x97, 0x9b, 0xea, 0x18, 0x5d, 0x47, 0x39, 0x81, 0xad, + 0xff, 0x36, 0x13, 0x7a, 0xb7, 0x5d, 0x3a, 0x7c, 0x85, 0x5e, 0x99, 0xd5, 0xa2, 0x9d, 0xd3, 0x22, + 0xfb, 0x56, 0x9e, 0xce, 0xbf, 0x95, 0x33, 0x41, 0x94, 0xdd, 0x8e, 0x3a, 0x99, 0x0f, 0xe1, 0x3c, + 0x57, 0x58, 0x62, 0x88, 0x2a, 0xb9, 0xf9, 0x4b, 0xe2, 0x9f, 0x67, 0xe0, 0x8d, 0x6a, 0xe2, 0x26, + 0xaf, 0x89, 0x8f, 0x40, 0x4b, 0xab, 0x75, 0x7e, 0xa5, 0x44, 0x0c, 0x7b, 0x41, 0x7a, 0xa9, 0xc8, + 0xbb, 0x67, 0x55, 0x95, 0xee, 0xfb, 0xc9, 0x7a, 0x72, 0xb3, 0x94, 0x4a, 0xfd, 0x76, 0xa9, 0xd4, + 0xe7, 0x02, 0x6c, 0xcc, 0xea, 0x04, 0xc8, 0xda, 0x65, 0xd5, 0xc6, 0xac, 0x4e, 0x40, 0x4a, 0x2c, + 0x04, 0x48, 0xaf, 0xe9, 0x29, 0x7c, 0x21, 0x60, 0x0d, 0x40, 0x95, 0x25, 0xb1, 0x9f, 0x3c, 0x5d, + 0xba, 0xb2, 0x28, 0x89, 0xfd, 0xda, 0xea, 0x6a, 0xae, 0xb6, 0xba, 0xca, 0x1f, 0x7f, 0xa7, 0x74, + 0x43, 0x7c, 0x01, 0x70, 0xc7, 0x89, 0x8e, 0xa4, 0x91, 0x79, 0x39, 0x67, 0x3b, 0xa1, 0x7a, 0x2f, + 0xf3, 0x4f, 0x0e, 0xc1, 0xae, 0xab, 0x4c, 0xc7, 0x3f, 0xb9, 0xfb, 0xc6, 0x11, 0xb1, 0x95, 0x75, + 0xc4, 0x37, 0x87, 0x1d, 0x84, 0x84, 0x28, 0x03, 0x88, 0x6f, 0xfd, 0xf7, 0x2d, 0xe8, 0x3e, 0x20, + 0x9e, 0xe2, 0xbc, 0x0e, 0xf0, 0x84, 0x86, 0x34, 0x66, 0x8e, 0x4f, 0x64, 0xf5, 0x39, 0x63, 0x64, + 0x20, 0x5f, 0x5f, 0x8e, 0x08, 0x4d, 0xe2, 0x1e, 0x28, 0x63, 0x8a, 0x6f, 0x0e, 0x3b, 0x24, 0x38, + 0x50, 0xf6, 0x13, 0xdf, 0x68, 0x19, 0x66, 0x22, 0x86, 0xad, 0x23, 0x61, 0xac, 0x69, 0x43, 0xfe, + 0xd0, 0x7d, 0xe8, 0xef, 0x3b, 0x24, 0x24, 0xca, 0xe1, 0x78, 0x59, 0x38, 0xc4, 0xd6, 0x11, 0x2f, + 0x94, 0xd9, 0x28, 0x20, 0xca, 0x14, 0x3d, 0x05, 0xdb, 0x1f, 0x05, 0x39, 0x14, 0x1f, 0x7b, 0x44, + 0xc5, 0x54, 0x82, 0xb2, 0x8b, 0xbd, 0x5c, 0x97, 0x49, 0xc5, 0x54, 0x12, 0x39, 0x26, 0x6c, 0xa8, + 0x62, 0xc4, 0x21, 0x21, 0xbf, 0x7a, 0xee, 0x60, 0xb6, 0x4f, 0x0d, 0xe2, 0xd1, 0x57, 0x54, 0x71, + 0x5c, 0x82, 0x8b, 0x27, 0x08, 0x50, 0x11, 0xfa, 0xef, 0x3e, 0xf4, 0x3f, 0x8b, 0x49, 0x38, 0xca, + 0xf4, 0x4b, 0x22, 0xa2, 0x58, 0x24, 0x0d, 0xbf, 0x0c, 0x84, 0xbb, 0xee, 0x41, 0x48, 0x3d, 0x33, + 0xed, 0x09, 0x9e, 0x11, 0x28, 0x3d, 0x0e, 0xbc, 0x27, 0xfb, 0x82, 0xe8, 0x63, 0x98, 0x3d, 0x70, + 0x5c, 0x46, 0x64, 0x17, 0xae, 0xb7, 0xfd, 0x56, 0xb9, 0xff, 0x97, 0x95, 0xb9, 0x75, 0x4f, 0x20, + 0x1b, 0x8a, 0x08, 0x0d, 0x61, 0xc9, 0xf1, 0x03, 0x51, 0x03, 0x86, 0x0e, 0x76, 0x9d, 0x17, 0xe3, + 0x17, 0x7f, 0x6f, 0xfb, 0xbd, 0x09, 0xbc, 0xee, 0x73, 0xca, 0xbd, 0x2c, 0xa1, 0x81, 0x9c, 0x12, + 0x0c, 0x11, 0x58, 0xa6, 0x31, 0x2b, 0x0b, 0x99, 0x11, 0x42, 0xb6, 0x27, 0x08, 0x79, 0x28, 0x48, + 0xf3, 0x52, 0x96, 0x68, 0x19, 0xa8, 0xed, 0xc2, 0xac, 0x54, 0x8e, 0x3b, 0xdd, 0x81, 0x43, 0xdc, + 0xa4, 0x8f, 0x29, 0x7f, 0x70, 0xf7, 0xa0, 0x01, 0x09, 0xb1, 0x6f, 0xab, 0x03, 0x4c, 0x7e, 0x72, + 0xfc, 0x63, 0xec, 0xc6, 0x24, 0x69, 0x64, 0x8a, 0x1f, 0xda, 0xdf, 0x66, 0x00, 0x95, 0x35, 0x4c, + 0xda, 0x18, 0x21, 0x89, 0xb8, 0x6b, 0x65, 0xfd, 0x75, 0x21, 0x03, 0x17, 0x3e, 0xfb, 0x13, 0xe8, + 0x5a, 0xd1, 0xb1, 0x29, 0x4c, 0x22, 0x64, 0xf6, 0xb6, 0x3f, 0x3c, 0xb5, 0x49, 0xb7, 0x76, 0xf6, + 0x1e, 0x09, 0xa8, 0xd1, 0xb1, 0xa2, 0x63, 0xf1, 0x85, 0x7e, 0x06, 0xf0, 0x8b, 0x88, 0xfa, 0x8a, + 0xb3, 0x3c, 0xf8, 0x8f, 0x4e, 0xcf, 0xf9, 0x47, 0x7b, 0x0f, 0x77, 0x25, 0xeb, 0x2e, 0x67, 0x27, + 0x79, 0x5b, 0x30, 0x1f, 0xe0, 0xf0, 0x69, 0x4c, 0x98, 0x62, 0x2f, 0x7d, 0xe1, 0xbb, 0xa7, 0x67, + 0xff, 0x63, 0xc9, 0x46, 0x4a, 0xe8, 0x07, 0x99, 0x5f, 0xda, 0x5f, 0xce, 0x40, 0x27, 0xd1, 0x8b, + 0x97, 0x91, 0xc2, 0xc3, 0xe5, 0x63, 0xca, 0x74, 0xfc, 0x03, 0xaa, 0x2c, 0x7a, 0x96, 0xc3, 0xe5, + 0x7b, 0xea, 0xbe, 0x7f, 0x40, 0xb9, 0xed, 0x43, 0x62, 0xd1, 0xd0, 0xe6, 0x97, 0xb6, 0xe3, 0x39, + 0xdc, 0xed, 0xe5, 0x59, 0x2e, 0x48, 0xf8, 0x9d, 0x04, 0x8c, 0x2e, 0xc3, 0x82, 0x38, 0xf6, 0x0c, + 0x66, 0x3b, 0xe1, 0x49, 0xdc, 0x0c, 0xe2, 0x15, 0x58, 0x7c, 0x1a, 0x53, 0x46, 0x4c, 0xeb, 0x10, + 0x87, 0xd8, 0x62, 0x34, 0x7d, 0xd6, 0x2c, 0x08, 0xf8, 0x4e, 0x0a, 0x46, 0xdf, 0x80, 0x15, 0x89, + 0x4a, 0x22, 0x0b, 0x07, 0x29, 0x05, 0x09, 0x55, 0xd5, 0xbb, 0x2c, 0x56, 0xef, 0x8a, 0xc5, 0x9d, + 0x64, 0x0d, 0x69, 0xd0, 0xb1, 0xa8, 0xe7, 0x11, 0x9f, 0x45, 0x22, 0x35, 0x76, 0x8d, 0xf4, 0x37, + 0xba, 0x05, 0x6b, 0xd8, 0x75, 0xe9, 0x33, 0x53, 0x50, 0xda, 0x66, 0x49, 0xbb, 0x39, 0x51, 0x94, + 0x68, 0x02, 0xe9, 0x33, 0x81, 0x63, 0xe4, 0x15, 0xd5, 0x2e, 0x40, 0x37, 0x3d, 0x47, 0x9e, 0x82, + 0x33, 0x0e, 0x29, 0xbe, 0xb5, 0xb3, 0xd0, 0xcf, 0x9e, 0x84, 0xf6, 0xaf, 0x36, 0x2c, 0x55, 0x04, + 0x15, 0x7a, 0x0c, 0xc0, 0xbd, 0x55, 0x86, 0x96, 0x72, 0xd7, 0xef, 0x9c, 0x3e, 0x38, 0xb9, 0xbf, + 0x4a, 0xb0, 0xc1, 0xbd, 0x5f, 0x7e, 0xa2, 0x9f, 0x43, 0x4f, 0x78, 0xac, 0xe2, 0x2e, 0x5d, 0xf6, + 0xe3, 0xaf, 0xc1, 0x9d, 0xeb, 0xaa, 0xd8, 0x8b, 0x18, 0x90, 0xdf, 0xda, 0x3f, 0x5a, 0xd0, 0x4d, + 0x05, 0xf3, 0xcb, 0x42, 0x1e, 0x94, 0x38, 0xeb, 0x28, 0xb9, 0x4f, 0x04, 0xec, 0x9e, 0x00, 0xfd, + 0x5f, 0xba, 0x92, 0xf6, 0x3e, 0xc0, 0x58, 0xff, 0x4a, 0x15, 0x5a, 0x95, 0x2a, 0xe8, 0x57, 0x60, + 0x9e, 0x5b, 0xd6, 0x21, 0xf6, 0x1e, 0x0b, 0x9d, 0x40, 0xdc, 0x95, 0x12, 0x27, 0x52, 0x15, 0x71, + 0xf2, 0x73, 0xfb, 0x8f, 0x03, 0xe8, 0x67, 0x5f, 0xf2, 0xe8, 0x4b, 0xe8, 0x65, 0x26, 0x4f, 0xe8, + 0xcd, 0xf2, 0xa1, 0x95, 0x27, 0x59, 0xda, 0x5b, 0x13, 0xb0, 0xd4, 0x95, 0x38, 0x85, 0x7c, 0x38, + 0x57, 0x1a, 0xdf, 0xa0, 0xab, 0x65, 0xea, 0xba, 0xe1, 0x90, 0x76, 0xad, 0x11, 0x6e, 0x2a, 0x8f, + 0xc1, 0x52, 0xc5, 0x3c, 0x06, 0xbd, 0x33, 0x81, 0x4b, 0x6e, 0x26, 0xa4, 0xbd, 0xdb, 0x10, 0x3b, + 0x95, 0xfa, 0x14, 0x50, 0x79, 0x58, 0x83, 0xae, 0x4d, 0x64, 0x33, 0x1e, 0x06, 0x69, 0xef, 0x34, + 0x43, 0xae, 0x55, 0x54, 0x8e, 0x71, 0x26, 0x2a, 0x9a, 0x1b, 0x14, 0x4d, 0x54, 0xb4, 0x30, 0x1b, + 0x9a, 0x42, 0x47, 0xb0, 0x58, 0x1c, 0xf1, 0xa0, 0x2b, 0x75, 0x23, 0xc9, 0xd2, 0x04, 0x49, 0xbb, + 0xda, 0x04, 0x35, 0x15, 0x46, 0xe0, 0x6c, 0x7e, 0xa4, 0x82, 0x2e, 0x97, 0xe9, 0x2b, 0x87, 0x4a, + 0xda, 0xe6, 0x64, 0xc4, 0xac, 0x4e, 0xc5, 0x31, 0x4b, 0x95, 0x4e, 0x35, 0x33, 0x9c, 0x2a, 0x9d, + 0xea, 0xa6, 0x36, 0xfa, 0x14, 0xfa, 0x65, 0xd2, 0xbb, 0x2f, 0x8c, 0x1f, 0xd0, 0x56, 0x1d, 0x9b, + 0xea, 0xf9, 0x87, 0x76, 0xbd, 0x31, 0x7e, 0x22, 0xfb, 0x46, 0x8b, 0xc7, 0x7a, 0x66, 0x0a, 0x51, + 0x15, 0xeb, 0xe5, 0xb9, 0x46, 0x55, 0xac, 0x57, 0x8d, 0x32, 0xa6, 0xd0, 0x10, 0xe6, 0x73, 0x73, + 0x09, 0xf4, 0x76, 0x1d, 0x65, 0xbe, 0xa1, 0xa1, 0x5d, 0x9e, 0x88, 0x97, 0xca, 0x30, 0x93, 0xec, + 0xa5, 0xd2, 0x55, 0xed, 0xe6, 0xf2, 0xf9, 0xea, 0xed, 0x49, 0x68, 0xb9, 0x50, 0x2e, 0x4d, 0x2f, + 0x2a, 0x43, 0xb9, 0x6e, 0x3a, 0x52, 0x19, 0xca, 0xf5, 0x03, 0x91, 0x29, 0xf4, 0x53, 0x80, 0xf1, + 0x84, 0x01, 0x5d, 0xaa, 0xa3, 0xce, 0x9e, 0xfe, 0x9b, 0x27, 0x23, 0xa5, 0xac, 0x9f, 0xc1, 0x72, + 0xd5, 0xc3, 0x1f, 0x55, 0x04, 0xfe, 0x09, 0xdd, 0x05, 0x6d, 0xab, 0x29, 0x7a, 0x2a, 0xf8, 0x73, + 0xe8, 0x24, 0xd3, 0x01, 0x74, 0xb1, 0x4c, 0x5d, 0x98, 0x87, 0x68, 0xfa, 0x49, 0x28, 0x19, 0x07, + 0xf6, 0x92, 0x58, 0x1d, 0xb7, 0xed, 0xeb, 0x63, 0xb5, 0x34, 0x60, 0xa8, 0x8f, 0xd5, 0xf2, 0x14, + 0x40, 0x88, 0x4b, 0x9d, 0x21, 0xdb, 0xe5, 0xae, 0x77, 0x86, 0x8a, 0x26, 0x7e, 0xbd, 0x33, 0x54, + 0x36, 0xce, 0xa7, 0xd0, 0xaf, 0x60, 0xa5, 0xba, 0xb9, 0x8d, 0x6a, 0x23, 0xbe, 0xa6, 0xc9, 0xae, + 0xdd, 0x68, 0x4e, 0x90, 0x8a, 0x7f, 0x91, 0xe4, 0xa7, 0x42, 0x73, 0xbb, 0x3e, 0x3f, 0x55, 0xb7, + 0xd8, 0xb5, 0xeb, 0x8d, 0xf1, 0xcb, 0xa1, 0x97, 0xed, 0x22, 0xd7, 0x5b, 0xbb, 0xa2, 0x61, 0x5e, + 0x6f, 0xed, 0xca, 0xc6, 0xb4, 0x88, 0x8f, 0xaa, 0x0e, 0x71, 0x55, 0x7c, 0x9c, 0xd0, 0xc2, 0xd6, + 0xb6, 0x9a, 0xa2, 0xe7, 0xae, 0xef, 0x72, 0x0b, 0x18, 0x4d, 0xdc, 0x7f, 0x2e, 0x33, 0xbf, 0xdb, + 0x10, 0xbb, 0xfe, 0x74, 0x93, 0x4c, 0x3d, 0x51, 0x81, 0x42, 0xc6, 0xbe, 0xde, 0x18, 0x3f, 0x95, + 0x1d, 0x24, 0x73, 0xdf, 0x4c, 0xfb, 0x16, 0x5d, 0x9d, 0xc0, 0x27, 0xd3, 0x7e, 0xd6, 0xae, 0x35, + 0xc2, 0xad, 0x8a, 0xde, 0x6c, 0x43, 0xf5, 0x24, 0x7f, 0x2a, 0x75, 0x81, 0x4f, 0xf2, 0xa7, 0x8a, + 0x1e, 0xed, 0x14, 0xfa, 0xcd, 0x78, 0xa0, 0x57, 0xee, 0x14, 0xa1, 0xed, 0xda, 0x5c, 0x50, 0xdb, + 0xb7, 0xd2, 0x6e, 0x9e, 0x8a, 0x26, 0xdd, 0xc8, 0xa7, 0x30, 0x23, 0xde, 0x5a, 0x68, 0xfd, 0xe4, + 0x47, 0x98, 0x76, 0xa1, 0x7a, 0x3d, 0x7d, 0x4a, 0x70, 0x4b, 0x0e, 0x67, 0xc5, 0xff, 0xdc, 0x6e, + 0xfe, 0x37, 0x00, 0x00, 0xff, 0xff, 0x62, 0x0a, 0xf6, 0x47, 0xfe, 0x26, 0x00, 0x00, } diff --git a/weed/replication/sink/s3sink/s3_sink.go b/weed/replication/sink/s3sink/s3_sink.go index d5cad3541..4cff341d0 100644 --- a/weed/replication/sink/s3sink/s3_sink.go +++ b/weed/replication/sink/s3sink/s3_sink.go @@ -56,7 +56,7 @@ func (s3sink *S3Sink) SetSourceFiler(s *source.FilerSource) { s3sink.filerSource = s } -func (s3sink *S3Sink) initialize(awsAccessKeyId, aswSecretAccessKey, region, bucket, dir string) error { +func (s3sink *S3Sink) initialize(awsAccessKeyId, awsSecretAccessKey, region, bucket, dir string) error { s3sink.region = region s3sink.bucket = bucket s3sink.dir = dir @@ -64,8 +64,8 @@ func (s3sink *S3Sink) initialize(awsAccessKeyId, aswSecretAccessKey, region, buc config := &aws.Config{ Region: aws.String(s3sink.region), } - if awsAccessKeyId != "" && aswSecretAccessKey != "" { - config.Credentials = credentials.NewStaticCredentials(awsAccessKeyId, aswSecretAccessKey, "") + if awsAccessKeyId != "" && awsSecretAccessKey != "" { + config.Credentials = credentials.NewStaticCredentials(awsAccessKeyId, awsSecretAccessKey, "") } sess, err := session.NewSession(config) diff --git a/weed/replication/sub/notification_aws_sqs.go b/weed/replication/sub/notification_aws_sqs.go index f0100f4de..bed26c79c 100644 --- a/weed/replication/sub/notification_aws_sqs.go +++ b/weed/replication/sub/notification_aws_sqs.go @@ -38,13 +38,13 @@ func (k *AwsSqsInput) Initialize(configuration util.Configuration) error { ) } -func (k *AwsSqsInput) initialize(awsAccessKeyId, aswSecretAccessKey, region, queueName string) (err error) { +func (k *AwsSqsInput) initialize(awsAccessKeyId, awsSecretAccessKey, region, queueName string) (err error) { config := &aws.Config{ Region: aws.String(region), } - if awsAccessKeyId != "" && aswSecretAccessKey != "" { - config.Credentials = credentials.NewStaticCredentials(awsAccessKeyId, aswSecretAccessKey, "") + if awsAccessKeyId != "" && awsSecretAccessKey != "" { + config.Credentials = credentials.NewStaticCredentials(awsAccessKeyId, awsSecretAccessKey, "") } sess, err := session.NewSession(config) diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index 3bf4aafac..d3bde66ee 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -14,7 +14,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "github.com/satori/go.uuid" + "github.com/google/uuid" ) type InitiateMultipartUploadResult struct { @@ -23,7 +23,7 @@ type InitiateMultipartUploadResult struct { } func (s3a *S3ApiServer) createMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (output *InitiateMultipartUploadResult, code ErrorCode) { - uploadId, _ := uuid.NewV4() + uploadId, _ := uuid.NewRandom() uploadIdString := uploadId.String() if err := s3a.mkdir(ctx, s3a.genUploadsFolder(*input.Bucket), uploadIdString, func(entry *filer_pb.Entry) { diff --git a/weed/server/common.go b/weed/server/common.go index d50c283f2..e3e16b927 100644 --- a/weed/server/common.go +++ b/weed/server/common.go @@ -14,6 +14,8 @@ import ( "github.com/chrislusf/seaweedfs/weed/storage/needle" "google.golang.org/grpc" + _ "github.com/chrislusf/seaweedfs/weed/storage/backend/s3_backend" + "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/stats" diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 9c35e8846..feab11c79 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -40,11 +40,16 @@ func (fs *FilerServer) ListEntries(ctx context.Context, req *filer_pb.ListEntrie limit = fs.option.DirListingLimit } + paginationLimit := 1024 + if limit < paginationLimit { + paginationLimit = limit + } + resp := &filer_pb.ListEntriesResponse{} lastFileName := req.StartFromFileName includeLastFile := req.InclusiveStartFrom for limit > 0 { - entries, err := fs.filer.ListDirectoryEntries(ctx, filer2.FullPath(req.Directory), lastFileName, includeLastFile, 1024) + entries, err := fs.filer.ListDirectoryEntries(ctx, filer2.FullPath(req.Directory), lastFileName, includeLastFile, paginationLimit) if err != nil { return nil, err } @@ -71,9 +76,12 @@ func (fs *FilerServer) ListEntries(ctx context.Context, req *filer_pb.ListEntrie Attributes: filer2.EntryAttributeToPb(entry), }) limit-- + if limit == 0 { + return resp, nil + } } - if len(resp.Entries) < 1024 { + if len(entries) < paginationLimit { break } diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index 5d95a5d7e..f94d83347 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -149,6 +149,16 @@ func (fs *FilerServer) updateFilerStore(ctx context.Context, r *http.Request, w stats.FilerRequestHistogram.WithLabelValues("postStoreWrite").Observe(time.Since(start).Seconds()) }() + modeStr := r.URL.Query().Get("mode") + if modeStr == "" { + modeStr = "0660" + } + mode, err := strconv.ParseUint(modeStr, 8, 32) + if err != nil { + glog.Errorf("Invalid mode format: %s, use 0660 by default", modeStr) + mode = 0660 + } + path := r.URL.Path if strings.HasSuffix(path, "/") { if ret.Name != "" { @@ -165,7 +175,7 @@ func (fs *FilerServer) updateFilerStore(ctx context.Context, r *http.Request, w Attr: filer2.Attr{ Mtime: time.Now(), Crtime: crTime, - Mode: 0660, + Mode: os.FileMode(mode), Uid: OS_UID, Gid: OS_GID, Replication: replication, diff --git a/weed/server/master_grpc_server.go b/weed/server/master_grpc_server.go index 82a190e39..a35374114 100644 --- a/weed/server/master_grpc_server.go +++ b/weed/server/master_grpc_server.go @@ -76,7 +76,9 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ int64(heartbeat.MaxVolumeCount)) glog.V(0).Infof("added volume server %v:%d", heartbeat.GetIp(), heartbeat.GetPort()) if err := stream.Send(&master_pb.HeartbeatResponse{ - VolumeSizeLimit: uint64(ms.option.VolumeSizeLimitMB) * 1024 * 1024, + VolumeSizeLimit: uint64(ms.option.VolumeSizeLimitMB) * 1024 * 1024, + MetricsAddress: ms.option.MetricsAddress, + MetricsIntervalSeconds: uint32(ms.option.MetricsIntervalSec), }); err != nil { glog.Warningf("SendHeartbeat.Send volume size to %s:%d %v", dn.Ip, dn.Port, err) return err @@ -164,9 +166,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ return err } if err := stream.Send(&master_pb.HeartbeatResponse{ - Leader: newLeader, - MetricsAddress: ms.option.MetricsAddress, - MetricsIntervalSeconds: uint32(ms.option.MetricsIntervalSec), + Leader: newLeader, }); err != nil { glog.Warningf("SendHeartbeat.Send response to to %s:%d %v", dn.Ip, dn.Port, err) return err diff --git a/weed/server/master_server.go b/weed/server/master_server.go index e68356c01..33a5129da 100644 --- a/weed/server/master_server.go +++ b/weed/server/master_server.go @@ -28,9 +28,8 @@ import ( ) const ( - MasterPrefix = "master.maintenance" - SequencerType = MasterPrefix + ".sequencer_type" - SequencerEtcdUrls = MasterPrefix + ".sequencer_etcd_urls" + SequencerType = "master.sequencer.type" + SequencerEtcdUrls = "master.sequencer.sequencer_etcd_urls" ) type MasterOption struct { @@ -194,8 +193,8 @@ func (ms *MasterServer) startAdminScripts() { v.SetDefault("master.maintenance.sleep_minutes", 17) sleepMinutes := v.GetInt("master.maintenance.sleep_minutes") - v.SetDefault("master.maintenance.filer_url", "http://localhost:8888/") - filerURL := v.GetString("master.maintenance.filer_url") + v.SetDefault("master.filer.default_filer_url", "http://localhost:8888/") + filerURL := v.GetString("master.filer.default_filer_url") scriptLines := strings.Split(adminScripts, "\n") @@ -207,7 +206,7 @@ func (ms *MasterServer) startAdminScripts() { shellOptions.FilerHost, shellOptions.FilerPort, shellOptions.Directory, err = util.ParseFilerUrl(filerURL) if err != nil { - glog.V(0).Infof("failed to parse master.maintenance.filer_url=%s : %v\n", filerURL, err) + glog.V(0).Infof("failed to parse master.filer.default_filer_urll=%s : %v\n", filerURL, err) return } @@ -251,14 +250,13 @@ func (ms *MasterServer) startAdminScripts() { func (ms *MasterServer) createSequencer(option *MasterOption) sequence.Sequencer { var seq sequence.Sequencer - seqType := strings.ToLower(util.Config().GetString(SequencerType)) - glog.V(0).Infof("[%s] : [%s]", SequencerType, seqType) + v := viper.GetViper() + seqType := strings.ToLower(v.GetString(SequencerType)) + glog.V(1).Infof("[%s] : [%s]", SequencerType, seqType) switch strings.ToLower(seqType) { - case "memory": - seq = sequence.NewMemorySequencer() case "etcd": var err error - urls := util.Config().GetString(SequencerEtcdUrls) + urls := v.GetString(SequencerEtcdUrls) glog.V(0).Infof("[%s] : [%s]", SequencerEtcdUrls, urls) seq, err = sequence.NewEtcdSequencer(urls, option.MetaFolder) if err != nil { diff --git a/weed/server/volume_grpc_copy_incremental.go b/weed/server/volume_grpc_copy_incremental.go index 6c5bb8a62..6d6c3daa3 100644 --- a/weed/server/volume_grpc_copy_incremental.go +++ b/weed/server/volume_grpc_copy_incremental.go @@ -47,7 +47,7 @@ func (vs *VolumeServer) VolumeSyncStatus(ctx context.Context, req *volume_server } -func sendFileContent(datBackend backend.DataStorageBackend, buf []byte, startOffset, stopOffset int64, stream volume_server_pb.VolumeServer_VolumeIncrementalCopyServer) error { +func sendFileContent(datBackend backend.BackendStorageFile, buf []byte, startOffset, stopOffset int64, stream volume_server_pb.VolumeServer_VolumeIncrementalCopyServer) error { var blockSizeLimit = int64(len(buf)) for i := int64(0); i < stopOffset-startOffset; i += blockSizeLimit { n, readErr := datBackend.ReadAt(buf, startOffset+i) diff --git a/weed/server/volume_grpc_tier.go b/weed/server/volume_grpc_tier.go new file mode 100644 index 000000000..cbd344ed0 --- /dev/null +++ b/weed/server/volume_grpc_tier.go @@ -0,0 +1,38 @@ +package weed_server + +import ( + "context" + "fmt" + "os" + + "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" + "github.com/chrislusf/seaweedfs/weed/storage/backend" + "github.com/chrislusf/seaweedfs/weed/storage/needle" +) + +// VolumeTierCopyDatToRemote copy dat file to a remote tier +func (vs *VolumeServer) VolumeTierCopyDatToRemote(ctx context.Context, req *volume_server_pb.VolumeTierCopyDatToRemoteRequest) (*volume_server_pb.VolumeTierCopyDatToRemoteResponse, error) { + + v := vs.store.GetVolume(needle.VolumeId(req.VolumeId)) + if v == nil { + return nil, fmt.Errorf("volume %d not found", req.VolumeId) + } + + if v.Collection != req.Collection { + return nil, fmt.Errorf("existing collection:%v unexpected input: %v", v.Collection, req.Collection) + } + + diskFile, ok := v.DataBackend.(*backend.DiskFile) + if !ok { + return nil, fmt.Errorf("volume %d is not on local disk", req.VolumeId) + } + err := uploadFileToRemote(ctx, req, diskFile.File) + + return &volume_server_pb.VolumeTierCopyDatToRemoteResponse{}, err +} + +func uploadFileToRemote(ctx context.Context, req *volume_server_pb.VolumeTierCopyDatToRemoteRequest, f *os.File) error { + println("copying dat file of", f.Name(), "to remote") + + return nil +} diff --git a/weed/shell/command_ec_encode.go b/weed/shell/command_ec_encode.go index f07cb93f9..4598b4682 100644 --- a/weed/shell/command_ec_encode.go +++ b/weed/shell/command_ec_encode.go @@ -95,7 +95,7 @@ func doEcEncode(ctx context.Context, commandEnv *CommandEnv, collection string, // mark the volume as readonly err = markVolumeReadonly(ctx, commandEnv.option.GrpcDialOption, needle.VolumeId(vid), locations) if err != nil { - return fmt.Errorf("generate ec shards for volume %d on %s: %v", vid, locations[0].Url, err) + return fmt.Errorf("mark volume %d as readonly on %s: %v", vid, locations[0].Url, err) } // generate ec shards diff --git a/weed/shell/command_volume_balance.go b/weed/shell/command_volume_balance.go index d7ef0d005..faece77d2 100644 --- a/weed/shell/command_volume_balance.go +++ b/weed/shell/command_volume_balance.go @@ -27,7 +27,7 @@ func (c *commandVolumeBalance) Name() string { func (c *commandVolumeBalance) Help() string { return `balance all volumes among volume servers - volume.balance [-c ALL|EACH_COLLECTION|<collection_name>] [-force] [-dataCenter=<data_center_name>] + volume.balance [-collection ALL|EACH_COLLECTION|<collection_name>] [-force] [-dataCenter=<data_center_name>] Algorithm: diff --git a/weed/shell/command_volume_tier.go b/weed/shell/command_volume_tier.go new file mode 100644 index 000000000..177d3f30d --- /dev/null +++ b/weed/shell/command_volume_tier.go @@ -0,0 +1,112 @@ +package shell + +import ( + "context" + "flag" + "fmt" + "io" + "time" + + "github.com/chrislusf/seaweedfs/weed/operation" + "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" + "github.com/chrislusf/seaweedfs/weed/storage/needle" + "google.golang.org/grpc" +) + +func init() { + Commands = append(Commands, &commandVolumeTier{}) +} + +type commandVolumeTier struct { +} + +func (c *commandVolumeTier) Name() string { + return "volume.tier" +} + +func (c *commandVolumeTier) Help() string { + return `copy the dat file of a volume to a remote tier + + ec.encode [-collection=""] [-fullPercent=95] [-quietFor=1h] + ec.encode [-collection=""] [-volumeId=<volume_id>] + + This command will: + 1. freeze one volume + 2. copy the dat file of a volume to a remote tier + +` +} + +func (c *commandVolumeTier) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + + tierCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + volumeId := tierCommand.Int("volumeId", 0, "the volume id") + collection := tierCommand.String("collection", "", "the collection name") + fullPercentage := tierCommand.Float64("fullPercent", 95, "the volume reaches the percentage of max volume size") + quietPeriod := tierCommand.Duration("quietFor", 24*time.Hour, "select volumes without no writes for this period") + dest := tierCommand.String("destination", "", "the target tier name") + if err = tierCommand.Parse(args); err != nil { + return nil + } + + ctx := context.Background() + vid := needle.VolumeId(*volumeId) + + // volumeId is provided + if vid != 0 { + return doVolumeTier(ctx, commandEnv, *collection, vid, *dest) + } + + // apply to all volumes in the collection + // reusing collectVolumeIdsForEcEncode for now + volumeIds, err := collectVolumeIdsForEcEncode(ctx, commandEnv, *collection, *fullPercentage, *quietPeriod) + if err != nil { + return err + } + fmt.Printf("tiering volumes: %v\n", volumeIds) + for _, vid := range volumeIds { + if err = doVolumeTier(ctx, commandEnv, *collection, vid, *dest); err != nil { + return err + } + } + + return nil +} + +func doVolumeTier(ctx context.Context, commandEnv *CommandEnv, collection string, vid needle.VolumeId, dest string) (err error) { + // find volume location + locations, found := commandEnv.MasterClient.GetLocations(uint32(vid)) + if !found { + return fmt.Errorf("volume %d not found", vid) + } + + // mark the volume as readonly + /* + err = markVolumeReadonly(ctx, commandEnv.option.GrpcDialOption, needle.VolumeId(vid), locations) + if err != nil { + return fmt.Errorf("mark volume %d as readonly on %s: %v", vid, locations[0].Url, err) + } + */ + + // copy the .dat file to remote tier + err = copyDatToRemoteTier(ctx, commandEnv.option.GrpcDialOption, needle.VolumeId(vid), collection, locations[0].Url, dest) + if err != nil { + return fmt.Errorf("copy dat file for volume %d on %s to %s: %v", vid, locations[0].Url, dest, err) + } + + return nil +} + +func copyDatToRemoteTier(ctx context.Context, grpcDialOption grpc.DialOption, volumeId needle.VolumeId, collection string, sourceVolumeServer string, dest string) error { + + err := operation.WithVolumeServerClient(sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + _, copyErr := volumeServerClient.VolumeTierCopyDatToRemote(ctx, &volume_server_pb.VolumeTierCopyDatToRemoteRequest{ + VolumeId: uint32(volumeId), + Collection: collection, + }) + return copyErr + }) + + return err + +} diff --git a/weed/storage/backend/backend.go b/weed/storage/backend/backend.go index ae0f84216..05312b74e 100644 --- a/weed/storage/backend/backend.go +++ b/weed/storage/backend/backend.go @@ -2,14 +2,62 @@ package backend import ( "io" + "os" "time" + + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/spf13/viper" ) -type DataStorageBackend interface { +type BackendStorageFile interface { io.ReaderAt io.WriterAt Truncate(off int64) error io.Closer GetStat() (datSize int64, modTime time.Time, err error) String() string + Instantiate(src *os.File) error +} + +type BackendStorage interface { + Name() string + NewStorageFile(key string) BackendStorageFile +} + +type StorageType string +type BackendStorageFactory interface { + StorageType() StorageType + BuildStorage(configuration util.Configuration, id string) (BackendStorage, error) +} + +var ( + BackendStorageFactories = make(map[StorageType]BackendStorageFactory) + BackendStorages = make(map[string]BackendStorage) +) + +func LoadConfiguration(config *viper.Viper) { + + StorageBackendPrefix := "storage.backend" + + backendSub := config.Sub(StorageBackendPrefix) + + for backendTypeName, _ := range config.GetStringMap(StorageBackendPrefix) { + backendStorageFactory, found := BackendStorageFactories[StorageType(backendTypeName)] + if !found { + glog.Fatalf("backend storage type %s not found", backendTypeName) + } + backendTypeSub := backendSub.Sub(backendTypeName) + for backendStorageId, _ := range backendSub.GetStringMap(backendTypeName) { + backendStorage, buildErr := backendStorageFactory.BuildStorage(backendTypeSub.Sub(backendStorageId), backendStorageId) + if buildErr != nil { + glog.Fatalf("fail to create backend storage %s.%s", backendTypeName, backendStorageId) + } + BackendStorages[backendTypeName+"."+backendStorageId] = backendStorage + if backendStorageId == "default" { + BackendStorages[backendTypeName] = backendStorage + } + } + } + } diff --git a/weed/storage/backend/disk_file.go b/weed/storage/backend/disk_file.go index 7f2b39d15..4929f0573 100644 --- a/weed/storage/backend/disk_file.go +++ b/weed/storage/backend/disk_file.go @@ -6,7 +6,7 @@ import ( ) var ( - _ DataStorageBackend = &DiskFile{} + _ BackendStorageFile = &DiskFile{} ) type DiskFile struct { @@ -48,3 +48,7 @@ func (df *DiskFile) GetStat() (datSize int64, modTime time.Time, err error) { func (df *DiskFile) String() string { return df.fullFilePath } + +func (df *DiskFile) Instantiate(src *os.File) error { + panic("should not implement Instantiate for DiskFile") +} diff --git a/weed/storage/backend/memory_map/memory_map_backend.go b/weed/storage/backend/memory_map/memory_map_backend.go index 57e3faedd..f1d723c3b 100644 --- a/weed/storage/backend/memory_map/memory_map_backend.go +++ b/weed/storage/backend/memory_map/memory_map_backend.go @@ -8,7 +8,7 @@ import ( ) var ( - _ backend.DataStorageBackend = &MemoryMappedFile{} + _ backend.BackendStorageFile = &MemoryMappedFile{} ) type MemoryMappedFile struct { @@ -50,7 +50,7 @@ func (mmf *MemoryMappedFile) Close() error { func (mmf *MemoryMappedFile) GetStat() (datSize int64, modTime time.Time, err error) { stat, e := mmf.mm.File.Stat() if e == nil { - return mmf.mm.End_of_file+1, stat.ModTime(), nil + return mmf.mm.End_of_file + 1, stat.ModTime(), nil } return 0, time.Time{}, err } @@ -58,3 +58,7 @@ func (mmf *MemoryMappedFile) GetStat() (datSize int64, modTime time.Time, err er func (mmf *MemoryMappedFile) String() string { return mmf.mm.File.Name() } + +func (mmf *MemoryMappedFile) Instantiate(src *os.File) error { + panic("should not implement Instantiate for MemoryMappedFile") +} diff --git a/weed/storage/backend/s3_backend/s3_backend.go b/weed/storage/backend/s3_backend/s3_backend.go new file mode 100644 index 000000000..980e9e9d7 --- /dev/null +++ b/weed/storage/backend/s3_backend/s3_backend.go @@ -0,0 +1,131 @@ +package s3_backend + +import ( + "fmt" + "os" + "strings" + "time" + + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3iface" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/storage/backend" + "github.com/chrislusf/seaweedfs/weed/util" +) + +func init() { + backend.BackendStorageFactories["s3"] = &S3BackendFactory{} +} + +type S3BackendFactory struct { +} + +func (factory *S3BackendFactory) StorageType() backend.StorageType { + return backend.StorageType("s3") +} +func (factory *S3BackendFactory) BuildStorage(configuration util.Configuration, id string) (backend.BackendStorage, error) { + return newS3BackendStorage(configuration, id) +} + +type S3BackendStorage struct { + id string + conn s3iface.S3API + region string + bucket string +} + +func newS3BackendStorage(configuration util.Configuration, id string) (s *S3BackendStorage, err error) { + s = &S3BackendStorage{} + s.id = id + s.conn, err = createSession( + configuration.GetString("aws_access_key_id"), + configuration.GetString("aws_secret_access_key"), + configuration.GetString("region")) + s.region = configuration.GetString("region") + s.bucket = configuration.GetString("bucket") + + glog.V(0).Infof("created s3 backend storage %s for region %s bucket %s", s.Name(), s.region, s.bucket) + return +} + +func (s *S3BackendStorage) Name() string { + return "s3." + s.id +} + +func (s *S3BackendStorage) NewStorageFile(key string) backend.BackendStorageFile { + if strings.HasPrefix(key, "/") { + key = key[1:] + } + + f := &S3BackendStorageFile{ + backendStorage: s, + key: key, + } + + return f +} + +type S3BackendStorageFile struct { + backendStorage *S3BackendStorage + key string +} + +func (s3backendStorageFile S3BackendStorageFile) ReadAt(p []byte, off int64) (n int, err error) { + bytesRange := fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p))-1) + getObjectOutput, getObjectErr := s3backendStorageFile.backendStorage.conn.GetObject(&s3.GetObjectInput{ + Bucket: &s3backendStorageFile.backendStorage.bucket, + Key: &s3backendStorageFile.key, + Range: &bytesRange, + }) + + if getObjectErr != nil { + return 0, fmt.Errorf("bucket %s GetObject %s: %v", + s3backendStorageFile.backendStorage.bucket, s3backendStorageFile.key, getObjectErr) + } + defer getObjectOutput.Body.Close() + + return getObjectOutput.Body.Read(p) + +} + +func (s3backendStorageFile S3BackendStorageFile) WriteAt(p []byte, off int64) (n int, err error) { + panic("implement me") +} + +func (s3backendStorageFile S3BackendStorageFile) Truncate(off int64) error { + panic("implement me") +} + +func (s3backendStorageFile S3BackendStorageFile) Close() error { + return nil +} + +func (s3backendStorageFile S3BackendStorageFile) GetStat() (datSize int64, modTime time.Time, err error) { + + headObjectOutput, headObjectErr := s3backendStorageFile.backendStorage.conn.HeadObject(&s3.HeadObjectInput{ + Bucket: &s3backendStorageFile.backendStorage.bucket, + Key: &s3backendStorageFile.key, + }) + + if headObjectErr != nil { + return 0, time.Now(), fmt.Errorf("bucket %s HeadObject %s: %v", + s3backendStorageFile.backendStorage.bucket, s3backendStorageFile.key, headObjectErr) + } + + datSize = int64(*headObjectOutput.ContentLength) + modTime = *headObjectOutput.LastModified + + return +} + +func (s3backendStorageFile S3BackendStorageFile) String() string { + return s3backendStorageFile.key +} + +func (s3backendStorageFile *S3BackendStorageFile) GetName() string { + return "s3" +} + +func (s3backendStorageFile S3BackendStorageFile) Instantiate(src *os.File) error { + panic("implement me") +} diff --git a/weed/storage/backend/s3_backend/s3_sessions.go b/weed/storage/backend/s3_backend/s3_sessions.go new file mode 100644 index 000000000..cd7b7ad47 --- /dev/null +++ b/weed/storage/backend/s3_backend/s3_sessions.go @@ -0,0 +1,54 @@ +package s3_backend + +import ( + "fmt" + "sync" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3iface" +) + +var ( + s3Sessions = make(map[string]s3iface.S3API) + sessionsLock sync.RWMutex +) + +func getSession(region string) (s3iface.S3API, bool) { + sessionsLock.RLock() + defer sessionsLock.RUnlock() + + sess, found := s3Sessions[region] + return sess, found +} + +func createSession(awsAccessKeyId, awsSecretAccessKey, region string) (s3iface.S3API, error) { + + sessionsLock.Lock() + defer sessionsLock.Unlock() + + if t, found := s3Sessions[region]; found { + return t, nil + } + + config := &aws.Config{ + Region: aws.String(region), + } + if awsAccessKeyId != "" && awsSecretAccessKey != "" { + config.Credentials = credentials.NewStaticCredentials(awsAccessKeyId, awsSecretAccessKey, "") + } + + sess, err := session.NewSession(config) + if err != nil { + return nil, fmt.Errorf("create aws session in region %s: %v", region, err) + } + + t := s3.New(sess) + + s3Sessions[region] = t + + return t, nil + +} diff --git a/weed/storage/backend/s3_backend/s3_upload.go b/weed/storage/backend/s3_backend/s3_upload.go new file mode 100644 index 000000000..1fde0a84f --- /dev/null +++ b/weed/storage/backend/s3_backend/s3_upload.go @@ -0,0 +1,56 @@ +package s3_backend + +import ( + "fmt" + "os" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3/s3iface" + "github.com/aws/aws-sdk-go/service/s3/s3manager" +) + +func uploadToS3(sess s3iface.S3API, filename string, destBucket string, destKey string) error { + + //open the file + f, err := os.Open(filename) + if err != nil { + return fmt.Errorf("failed to open file %q, %v", filename, err) + } + defer f.Close() + + info, err := f.Stat() + if err != nil { + return fmt.Errorf("failed to stat file %q, %v", filename, err) + } + + fileSize := info.Size() + + partSize := int64(64 * 1024 * 1024) // The minimum/default allowed part size is 5MB + for partSize*1000 < fileSize { + partSize *= 4 + } + + // Create an uploader with the session and custom options + uploader := s3manager.NewUploaderWithClient(sess, func(u *s3manager.Uploader) { + u.PartSize = partSize + u.Concurrency = 15 // default is 15 + }) + + // Upload the file to S3. + result, err := uploader.Upload(&s3manager.UploadInput{ + Bucket: aws.String(destBucket), + Key: aws.String(destKey), + Body: f, + ACL: aws.String("private"), + ServerSideEncryption: aws.String("AES256"), + StorageClass: aws.String("STANDARD_IA"), + }) + + //in case it fails to upload + if err != nil { + return fmt.Errorf("failed to upload file, %v", err) + } + fmt.Printf("file %s uploaded to %s\n", filename, result.Location) + + return nil +} diff --git a/weed/storage/needle/needle_read_write.go b/weed/storage/needle/needle_read_write.go index 8e5d18b1a..60301fa36 100644 --- a/weed/storage/needle/needle_read_write.go +++ b/weed/storage/needle/needle_read_write.go @@ -125,10 +125,10 @@ func (n *Needle) prepareWriteBuffer(version Version) ([]byte, uint32, int64, err return writeBytes, 0, 0, fmt.Errorf("Unsupported Version! (%d)", version) } -func (n *Needle) Append(w backend.DataStorageBackend, version Version) (offset uint64, size uint32, actualSize int64, err error) { +func (n *Needle) Append(w backend.BackendStorageFile, version Version) (offset uint64, size uint32, actualSize int64, err error) { if end, _, e := w.GetStat(); e == nil { - defer func(w backend.DataStorageBackend, off int64) { + defer func(w backend.BackendStorageFile, off int64) { if err != nil { if te := w.Truncate(end); te != nil { glog.V(0).Infof("Failed to truncate %s back to %d with error: %v", w.String(), end, te) @@ -150,7 +150,7 @@ func (n *Needle) Append(w backend.DataStorageBackend, version Version) (offset u return offset, size, actualSize, err } -func ReadNeedleBlob(r backend.DataStorageBackend, offset int64, size uint32, version Version) (dataSlice []byte, err error) { +func ReadNeedleBlob(r backend.BackendStorageFile, offset int64, size uint32, version Version) (dataSlice []byte, err error) { dataSize := GetActualSize(size, version) dataSlice = make([]byte, int(dataSize)) @@ -191,7 +191,7 @@ func (n *Needle) ReadBytes(bytes []byte, offset int64, size uint32, version Vers } // ReadData hydrates the needle from the file, with only n.Id is set. -func (n *Needle) ReadData(r backend.DataStorageBackend, offset int64, size uint32, version Version) (err error) { +func (n *Needle) ReadData(r backend.BackendStorageFile, offset int64, size uint32, version Version) (err error) { bytes, err := ReadNeedleBlob(r, offset, size, version) if err != nil { return err @@ -266,7 +266,7 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) { return nil } -func ReadNeedleHeader(r backend.DataStorageBackend, version Version, offset int64) (n *Needle, bytes []byte, bodyLength int64, err error) { +func ReadNeedleHeader(r backend.BackendStorageFile, version Version, offset int64) (n *Needle, bytes []byte, bodyLength int64, err error) { n = new(Needle) if version == Version1 || version == Version2 || version == Version3 { bytes = make([]byte, NeedleHeaderSize) @@ -301,7 +301,7 @@ func NeedleBodyLength(needleSize uint32, version Version) int64 { //n should be a needle already read the header //the input stream will read until next file entry -func (n *Needle) ReadNeedleBody(r backend.DataStorageBackend, version Version, offset int64, bodyLength int64) (bytes []byte, err error) { +func (n *Needle) ReadNeedleBody(r backend.BackendStorageFile, version Version, offset int64, bodyLength int64) (bytes []byte, err error) { if bodyLength <= 0 { return nil, nil diff --git a/weed/storage/store_ec.go b/weed/storage/store_ec.go index 7e3f1a46c..208560b7e 100644 --- a/weed/storage/store_ec.go +++ b/weed/storage/store_ec.go @@ -120,8 +120,13 @@ func (s *Store) ReadEcShardNeedle(ctx context.Context, vid needle.VolumeId, n *n if localEcVolume, found := location.FindEcVolume(vid); found { // read the volume version + readCounter := 0 for localEcVolume.Version == 0 { err := s.readEcVolumeVersion(ctx, vid, localEcVolume) + readCounter++ + if readCounter > 10 && err != nil { + return 0, fmt.Errorf("fail to read ec volume %d: %v", vid, err) + } time.Sleep(1357 * time.Millisecond) glog.V(0).Infof("ReadEcShardNeedle vid %d version:%v: %v", vid, localEcVolume.Version, err) } diff --git a/weed/storage/volume.go b/weed/storage/volume.go index e85696eab..00d76a4b3 100644 --- a/weed/storage/volume.go +++ b/weed/storage/volume.go @@ -21,7 +21,7 @@ type Volume struct { Id needle.VolumeId dir string Collection string - DataBackend backend.DataStorageBackend + DataBackend backend.BackendStorageFile nm NeedleMapper needleMapKind NeedleMapType readOnly bool diff --git a/weed/storage/volume_checking.go b/weed/storage/volume_checking.go index 61b59e9f7..a65c2a3ff 100644 --- a/weed/storage/volume_checking.go +++ b/weed/storage/volume_checking.go @@ -55,7 +55,7 @@ func readIndexEntryAtOffset(indexFile *os.File, offset int64) (bytes []byte, err return } -func verifyNeedleIntegrity(datFile backend.DataStorageBackend, v needle.Version, offset int64, key NeedleId, size uint32) (lastAppendAtNs uint64, err error) { +func verifyNeedleIntegrity(datFile backend.BackendStorageFile, v needle.Version, offset int64, key NeedleId, size uint32) (lastAppendAtNs uint64, err error) { n := new(needle.Needle) if err = n.ReadData(datFile, offset, size, v); err != nil { return n.AppendAtNs, err diff --git a/weed/storage/volume_create.go b/weed/storage/volume_create.go index b27a62990..ee41c50a9 100644 --- a/weed/storage/volume_create.go +++ b/weed/storage/volume_create.go @@ -9,7 +9,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/storage/backend" ) -func createVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32) (backend.DataStorageBackend, error) { +func createVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32) (backend.BackendStorageFile, error) { file, e := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) if preallocate > 0 { glog.V(0).Infof("Preallocated disk space for %s is not supported", fileName) diff --git a/weed/storage/volume_create_linux.go b/weed/storage/volume_create_linux.go index e3305d991..5fafbe924 100644 --- a/weed/storage/volume_create_linux.go +++ b/weed/storage/volume_create_linux.go @@ -10,7 +10,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/storage/backend" ) -func createVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32) (backend.DataStorageBackend, error) { +func createVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32) (backend.BackendStorageFile, error) { file, e := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) if preallocate != 0 { syscall.Fallocate(int(file.Fd()), 1, 0, preallocate) diff --git a/weed/storage/volume_create_windows.go b/weed/storage/volume_create_windows.go index 81536810b..9e5d8f87d 100644 --- a/weed/storage/volume_create_windows.go +++ b/weed/storage/volume_create_windows.go @@ -11,7 +11,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/storage/backend/memory_map/os_overloads" ) -func createVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32) (backend.DataStorageBackend, error) { +func createVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32) (backend.BackendStorageFile, error) { if preallocate > 0 { glog.V(0).Infof("Preallocated disk space for %s is not supported", fileName) diff --git a/weed/storage/volume_read_write.go b/weed/storage/volume_read_write.go index d9b79795b..c1758c18b 100644 --- a/weed/storage/volume_read_write.go +++ b/weed/storage/volume_read_write.go @@ -50,8 +50,8 @@ func (v *Volume) Destroy() (err error) { os.Remove(v.FileName() + ".idx") os.Remove(v.FileName() + ".cpd") os.Remove(v.FileName() + ".cpx") - os.Remove(v.FileName() + ".ldb") - os.Remove(v.FileName() + ".bdb") + os.RemoveAll(v.FileName() + ".ldb") + os.RemoveAll(v.FileName() + ".bdb") return } @@ -195,7 +195,7 @@ func ScanVolumeFile(dirname string, collection string, id needle.VolumeId, return ScanVolumeFileFrom(version, v.DataBackend, offset, volumeFileScanner) } -func ScanVolumeFileFrom(version needle.Version, datBackend backend.DataStorageBackend, offset int64, volumeFileScanner VolumeFileScanner) (err error) { +func ScanVolumeFileFrom(version needle.Version, datBackend backend.BackendStorageFile, offset int64, volumeFileScanner VolumeFileScanner) (err error) { n, nh, rest, e := needle.ReadNeedleHeader(datBackend, version, offset) if e != nil { if e == io.EOF { diff --git a/weed/storage/volume_super_block.go b/weed/storage/volume_super_block.go index bce5af465..911cd5455 100644 --- a/weed/storage/volume_super_block.go +++ b/weed/storage/volume_super_block.go @@ -104,7 +104,7 @@ func (v *Volume) readSuperBlock() (err error) { } // ReadSuperBlock reads from data file and load it into volume's super block -func ReadSuperBlock(datBackend backend.DataStorageBackend) (superBlock SuperBlock, err error) { +func ReadSuperBlock(datBackend backend.BackendStorageFile) (superBlock SuperBlock, err error) { header := make([]byte, _SuperBlockSize) if _, e := datBackend.ReadAt(header, 0); e != nil { diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go index e90746b54..caa1777e4 100644 --- a/weed/storage/volume_vacuum.go +++ b/weed/storage/volume_vacuum.go @@ -132,7 +132,7 @@ func (v *Volume) cleanupCompact() error { return nil } -func fetchCompactRevisionFromDatFile(datBackend backend.DataStorageBackend) (compactRevision uint16, err error) { +func fetchCompactRevisionFromDatFile(datBackend backend.BackendStorageFile) (compactRevision uint16, err error) { superBlock, err := ReadSuperBlock(datBackend) if err != nil { return 0, err @@ -270,7 +270,7 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI type VolumeFileScanner4Vacuum struct { version needle.Version v *Volume - dstBackend backend.DataStorageBackend + dstBackend backend.BackendStorageFile nm *NeedleMap newOffset int64 now uint64 @@ -312,7 +312,7 @@ func (scanner *VolumeFileScanner4Vacuum) VisitNeedle(n *needle.Needle, offset in func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string, preallocate int64, compactionBytePerSecond int64) (err error) { var ( - dst backend.DataStorageBackend + dst backend.BackendStorageFile idx *os.File ) if dst, err = createVolumeFile(dstName, preallocate, 0); err != nil { diff --git a/weed/topology/topology_vacuum.go b/weed/topology/topology_vacuum.go index 37a6a30b9..ff32f1874 100644 --- a/weed/topology/topology_vacuum.go +++ b/weed/topology/topology_vacuum.go @@ -35,13 +35,12 @@ func batchVacuumVolumeCheck(grpcDialOption grpc.DialOption, vl *VolumeLayout, vi }(index, dn.Url(), vid) } isCheckSuccess := true - for _ = range locationlist.list { + for range locationlist.list { select { case canVacuum := <-ch: isCheckSuccess = isCheckSuccess && canVacuum case <-time.After(30 * time.Minute): - isCheckSuccess = false - break + return false } } return isCheckSuccess @@ -71,13 +70,12 @@ func batchVacuumVolumeCompact(grpcDialOption grpc.DialOption, vl *VolumeLayout, }(index, dn.Url(), vid) } isVacuumSuccess := true - for _ = range locationlist.list { + for range locationlist.list { select { case canCommit := <-ch: isVacuumSuccess = isVacuumSuccess && canCommit case <-time.After(30 * time.Minute): - isVacuumSuccess = false - break + return false } } return isVacuumSuccess diff --git a/weed/util/config.go b/weed/util/config.go index 0ace53a37..84f146bc8 100644 --- a/weed/util/config.go +++ b/weed/util/config.go @@ -41,7 +41,3 @@ func LoadConfiguration(configFileName string, required bool) (loaded bool) { return true } - -func Config() Configuration { - return viper.GetViper() -} diff --git a/weed/util/constants.go b/weed/util/constants.go index 4f16a783c..f0df5fd59 100644 --- a/weed/util/constants.go +++ b/weed/util/constants.go @@ -5,5 +5,5 @@ import ( ) var ( - VERSION = fmt.Sprintf("%s %d.%d", sizeLimit, 1, 44) + VERSION = fmt.Sprintf("%s %d.%d", sizeLimit, 1, 45) ) |