diff options
36 files changed, 434 insertions, 252 deletions
diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 9ee552561..5983c84d8 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -99,6 +99,7 @@ message FileChunk { string source_file_id = 6; // to be deprecated FileId fid = 7; FileId source_fid = 8; + bytes cipher_key = 9; } message FileId { @@ -229,4 +230,5 @@ message GetFilerConfigurationResponse { uint32 max_mb = 4; string dir_buckets = 5; string dir_queues = 6; + bool cipher = 7; } diff --git a/unmaintained/repeated_vacuum/repeated_vacuum.go b/unmaintained/repeated_vacuum/repeated_vacuum.go index 718b6faa1..4a0464eda 100644 --- a/unmaintained/repeated_vacuum/repeated_vacuum.go +++ b/unmaintained/repeated_vacuum/repeated_vacuum.go @@ -35,7 +35,7 @@ func main() { targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid) - _, err = operation.Upload(targetUrl, fmt.Sprintf("test%d", i), reader, false, "", nil, assignResult.Auth) + _, err = operation.Upload(targetUrl, fmt.Sprintf("test%d", i), false, reader, false, "", nil, assignResult.Auth) if err != nil { log.Fatalf("upload: %v", err) } diff --git a/weed/command/filer.go b/weed/command/filer.go index b5b595215..327ee8316 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -34,6 +34,7 @@ type FilerOptions struct { dataCenter *string enableNotification *bool disableHttp *bool + cipher *bool // default leveldb directory, used in "weed server" mode defaultLevelDbDirectory *string @@ -53,6 +54,7 @@ func init() { f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size") f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to write to volumes in this data center") f.disableHttp = cmdFiler.Flag.Bool("disableHttp", false, "disable http request, only gRpc operations are allowed") + f.cipher = cmdFiler.Flag.Bool("encryptVolumeData", false, "encrypt data on volume servers") } var cmdFiler = &Command{ @@ -111,6 +113,7 @@ func (fo *FilerOptions) startFiler() { DefaultLevelDbDir: defaultLevelDbDirectory, DisableHttp: *fo.disableHttp, Port: uint32(*fo.port), + Cipher: *fo.cipher, }) if nfs_err != nil { glog.Fatalf("Filer startup error: %v", nfs_err) diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.go index 3e7ae1db2..8c01cfd74 100644 --- a/weed/command/filer_copy.go +++ b/weed/command/filer_copy.go @@ -41,6 +41,7 @@ type CopyOptions struct { compressionLevel *int grpcDialOption grpc.DialOption masters []string + cipher bool } func init() { @@ -108,7 +109,7 @@ func runCopy(cmd *Command, args []string) bool { filerGrpcAddress := fmt.Sprintf("%s:%d", filerUrl.Hostname(), filerGrpcPort) copy.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client") - masters, collection, replication, maxMB, err := readFilerConfiguration(copy.grpcDialOption, filerGrpcAddress) + masters, collection, replication, maxMB, cipher, err := readFilerConfiguration(copy.grpcDialOption, filerGrpcAddress) if err != nil { fmt.Printf("read from filer %s: %v\n", filerGrpcAddress, err) return false @@ -123,6 +124,7 @@ func runCopy(cmd *Command, args []string) bool { *copy.maxMB = int(maxMB) } copy.masters = masters + copy.cipher = cipher if *cmdCopy.IsDebug { util.SetupProfiling("filer.copy.cpu.pprof", "filer.copy.mem.pprof") @@ -159,13 +161,14 @@ func runCopy(cmd *Command, args []string) bool { return true } -func readFilerConfiguration(grpcDialOption grpc.DialOption, filerGrpcAddress string) (masters []string, collection, replication string, maxMB uint32, err error) { +func readFilerConfiguration(grpcDialOption grpc.DialOption, filerGrpcAddress string) (masters []string, collection, replication string, maxMB uint32, cipher bool, err error) { err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) if err != nil { return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err) } masters, collection, replication, maxMB = resp.Masters, resp.Collection, resp.Replication, resp.MaxMb + cipher = resp.Cipher return nil }) return @@ -300,7 +303,7 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err targetUrl := "http://" + assignResult.Url + "/" + assignResult.FileId - uploadResult, err := operation.UploadWithLocalCompressionLevel(targetUrl, fileName, f, false, mimeType, nil, security.EncodedJwt(assignResult.Auth), *worker.options.compressionLevel) + uploadResult, err := operation.UploadWithLocalCompressionLevel(targetUrl, fileName, worker.options.cipher, f, false, mimeType, nil, security.EncodedJwt(assignResult.Auth), *worker.options.compressionLevel) if err != nil { return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err) } @@ -310,11 +313,12 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err fmt.Printf("uploaded %s to %s\n", fileName, targetUrl) chunks = append(chunks, &filer_pb.FileChunk{ - FileId: assignResult.FileId, - Offset: 0, - Size: uint64(uploadResult.Size), - Mtime: time.Now().UnixNano(), - ETag: uploadResult.ETag, + FileId: assignResult.FileId, + Offset: 0, + Size: uint64(uploadResult.Size), + Mtime: time.Now().UnixNano(), + ETag: uploadResult.ETag, + CipherKey: uploadResult.CipherKey, }) fmt.Printf("copied %s => http://%s%s%s\n", fileName, worker.filerHost, task.destinationUrlPath, fileName) @@ -409,10 +413,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File, replication = assignResult.Replication } - uploadResult, err := operation.Upload(targetUrl, - fileName+"-"+strconv.FormatInt(i+1, 10), - io.NewSectionReader(f, i*chunkSize, chunkSize), - false, "", nil, security.EncodedJwt(assignResult.Auth)) + uploadResult, err := operation.Upload(targetUrl, fileName+"-"+strconv.FormatInt(i+1, 10), false, io.NewSectionReader(f, i*chunkSize, chunkSize), false, "", nil, security.EncodedJwt(assignResult.Auth)) if err != nil { uploadError = fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err) return @@ -422,11 +423,12 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File, return } chunksChan <- &filer_pb.FileChunk{ - FileId: assignResult.FileId, - Offset: i * chunkSize, - Size: uint64(uploadResult.Size), - Mtime: time.Now().UnixNano(), - ETag: uploadResult.ETag, + FileId: assignResult.FileId, + Offset: i * chunkSize, + Size: uint64(uploadResult.Size), + Mtime: time.Now().UnixNano(), + ETag: uploadResult.ETag, + CipherKey: uploadResult.CipherKey, } fmt.Printf("uploaded %s-%d to %s [%d,%d)\n", fileName, i+1, targetUrl, i*chunkSize, i*chunkSize+int64(uploadResult.Size)) }(i) diff --git a/weed/command/mount.go b/weed/command/mount.go index e73cbee10..792845083 100644 --- a/weed/command/mount.go +++ b/weed/command/mount.go @@ -54,7 +54,7 @@ var cmdMount = &Command{ On OS X, it requires OSXFUSE (http://osxfuse.github.com/). - If the SeaweedFS systemm runs in a container cluster, e.g. managed by kubernetes or docker compose, + If the SeaweedFS system runs in a container cluster, e.g. managed by kubernetes or docker compose, the volume servers are not accessible by their own ip addresses. In "outsideContainerClusterMode", the mount will use the filer ip address instead, assuming: * All volume server containers are accessible through the same hostname or IP address as the filer. diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index b195bf143..9177091a5 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -145,11 +145,13 @@ func RunMount(filer, filerMountRootPath, dir, collection, replication, dataCente // try to connect to filer, filerBucketsPath may be useful later grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") + var cipher bool err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { - _, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) + resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) if err != nil { return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err) } + cipher = resp.Cipher return nil }) if err != nil { @@ -183,6 +185,7 @@ func RunMount(filer, filerMountRootPath, dir, collection, replication, dataCente MountMtime: time.Now(), Umask: umask, OutsideContainerClusterMode: outsideContainerClusterMode, + Cipher: cipher, })) if err != nil { fuse.Unmount(dir) diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index 5b246b7c0..f4a08fb51 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -75,9 +75,9 @@ const ( # recursive_delete will delete all sub folders and files, similar to "rm -Rf" recursive_delete = false # directories under this folder will be automatically creating a separate bucket -buckets_folder = /buckets +buckets_folder = "/buckets" # directories under this folder will be store message queue data -queues_folder = /queues +queues_folder = "/queues" #################################################### # The following are filer store options diff --git a/weed/command/server.go b/weed/command/server.go index c9d27555c..f45429193 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -82,6 +82,7 @@ func init() { filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing") filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit") filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size") + filerOptions.cipher = cmdServer.Flag.Bool("filer.encryptVolumeData", false, "encrypt data on volume servers") serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port") serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port") diff --git a/weed/command/webdav.go b/weed/command/webdav.go index ba88a17be..4f5d5f5ce 100644 --- a/weed/command/webdav.go +++ b/weed/command/webdav.go @@ -1,6 +1,7 @@ package command import ( + "context" "fmt" "net/http" "os/user" @@ -9,6 +10,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/server" "github.com/chrislusf/seaweedfs/weed/util" @@ -55,12 +57,6 @@ func runWebDav(cmd *Command, args []string) bool { func (wo *WebDavOption) startWebDav() bool { - filerGrpcAddress, err := pb.ParseFilerGrpcAddress(*wo.filer) - if err != nil { - glog.Fatal(err) - return false - } - // detect current user uid, gid := uint32(0), uint32(0) if u, err := user.Current(); err == nil { @@ -72,13 +68,43 @@ func (wo *WebDavOption) startWebDav() bool { } } + // parse filer grpc address + filerGrpcAddress, err := pb.ParseFilerGrpcAddress(*wo.filer) + if err != nil { + glog.Fatal(err) + return false + } + + grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") + + var cipher bool + // connect to filer + for { + err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) + if err != nil { + return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err) + } + cipher = resp.Cipher + return nil + }) + if err != nil { + glog.V(0).Infof("wait to connect to filer %s grpc address %s", *wo.filer, filerGrpcAddress) + time.Sleep(time.Second) + } else { + glog.V(0).Infof("connected to filer %s grpc address %s", *wo.filer, filerGrpcAddress) + break + } + } + ws, webdavServer_err := weed_server.NewWebDavServer(&weed_server.WebDavOption{ Filer: *wo.filer, FilerGrpcAddress: filerGrpcAddress, - GrpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.client"), + GrpcDialOption: grpcDialOption, Collection: *wo.collection, Uid: uid, Gid: gid, + Cipher: cipher, }) if webdavServer_err != nil { glog.Fatalf("WebDav Server startup error: %v", webdavServer_err) diff --git a/weed/filer2/filechunks.go b/weed/filer2/filechunks.go index b5876df82..98a965337 100644 --- a/weed/filer2/filechunks.go +++ b/weed/filer2/filechunks.go @@ -71,6 +71,7 @@ type ChunkView struct { Size uint64 LogicOffset int64 IsFullChunk bool + CipherKey []byte } func ViewFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int) (views []*ChunkView) { @@ -94,6 +95,7 @@ func ViewFromVisibleIntervals(visibles []VisibleInterval, offset int64, size int Size: uint64(min(chunk.stop, stop) - offset), LogicOffset: offset, IsFullChunk: isFullChunk, + CipherKey: chunk.cipherKey, }) offset = min(chunk.stop, stop) } @@ -120,13 +122,7 @@ var bufPool = sync.Pool{ func MergeIntoVisibles(visibles, newVisibles []VisibleInterval, chunk *filer_pb.FileChunk) []VisibleInterval { - newV := newVisibleInterval( - chunk.Offset, - chunk.Offset+int64(chunk.Size), - chunk.GetFileIdString(), - chunk.Mtime, - true, - ) + newV := newVisibleInterval(chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.GetFileIdString(), chunk.Mtime, true, chunk.CipherKey) length := len(visibles) if length == 0 { @@ -140,23 +136,11 @@ func MergeIntoVisibles(visibles, newVisibles []VisibleInterval, chunk *filer_pb. logPrintf(" before", visibles) for _, v := range visibles { if v.start < chunk.Offset && chunk.Offset < v.stop { - newVisibles = append(newVisibles, newVisibleInterval( - v.start, - chunk.Offset, - v.fileId, - v.modifiedTime, - false, - )) + newVisibles = append(newVisibles, newVisibleInterval(v.start, chunk.Offset, v.fileId, v.modifiedTime, false, v.cipherKey)) } chunkStop := chunk.Offset + int64(chunk.Size) if v.start < chunkStop && chunkStop < v.stop { - newVisibles = append(newVisibles, newVisibleInterval( - chunkStop, - v.stop, - v.fileId, - v.modifiedTime, - false, - )) + newVisibles = append(newVisibles, newVisibleInterval(chunkStop, v.stop, v.fileId, v.modifiedTime, false, v.cipherKey)) } if chunkStop <= v.start || v.stop <= chunk.Offset { newVisibles = append(newVisibles, v) @@ -208,15 +192,17 @@ type VisibleInterval struct { modifiedTime int64 fileId string isFullChunk bool + cipherKey []byte } -func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, isFullChunk bool) VisibleInterval { +func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, isFullChunk bool, cipherKey []byte) VisibleInterval { return VisibleInterval{ start: start, stop: stop, fileId: fileId, modifiedTime: modifiedTime, isFullChunk: isFullChunk, + cipherKey: cipherKey, } } diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index bf43c3c52..cbb14a5c1 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -33,6 +33,7 @@ type Filer struct { DirBucketsPath string DirQueuesPath string buckets *FilerBuckets + Cipher bool } func NewFiler(masters []string, grpcDialOption grpc.DialOption, filerGrpcPort uint32) *Filer { diff --git a/weed/filer2/filer_client_util.go b/weed/filer2/filer_client_util.go index ab9db2992..e80c4bf36 100644 --- a/weed/filer2/filer_client_util.go +++ b/weed/filer2/filer_client_util.go @@ -70,12 +70,7 @@ func ReadIntoBuffer(filerClient FilerClient, fullFilePath FullPath, buff []byte, volumeServerAddress := filerClient.AdjustedUrl(locations.Locations[0].Url) var n int64 - n, err = util.ReadUrl( - fmt.Sprintf("http://%s/%s", volumeServerAddress, chunkView.FileId), - chunkView.Offset, - int(chunkView.Size), - buff[chunkView.LogicOffset-baseOffset:chunkView.LogicOffset-baseOffset+int64(chunkView.Size)], - !chunkView.IsFullChunk) + n, err = util.ReadUrl(fmt.Sprintf("http://%s/%s", volumeServerAddress, chunkView.FileId), chunkView.CipherKey, chunkView.IsFullChunk, chunkView.Offset, int(chunkView.Size), buff[chunkView.LogicOffset-baseOffset:chunkView.LogicOffset-baseOffset+int64(chunkView.Size)]) if err != nil { diff --git a/weed/filer2/redis/universal_redis_store.go b/weed/filer2/redis/universal_redis_store.go index 62257e91e..2162be733 100644 --- a/weed/filer2/redis/universal_redis_store.go +++ b/weed/filer2/redis/universal_redis_store.go @@ -3,12 +3,14 @@ package redis import ( "context" "fmt" - "github.com/chrislusf/seaweedfs/weed/filer2" - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/go-redis/redis" "sort" "strings" "time" + + "github.com/go-redis/redis" + + "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/glog" ) const ( @@ -120,7 +122,8 @@ func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, full func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { - members, err := store.Client.SMembers(genDirectoryListKey(string(fullpath))).Result() + dirListKey := genDirectoryListKey(string(fullpath)) + members, err := store.Client.SMembers(dirListKey).Result() if err != nil { return nil, fmt.Errorf("list %s : %v", fullpath, err) } @@ -159,6 +162,13 @@ func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, full if err != nil { glog.V(0).Infof("list %s : %v", path, err) } else { + if entry.TtlSec > 0 { + if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) { + store.Client.Del(string(path)).Result() + store.Client.SRem(dirListKey, fileName).Result() + continue + } + } entries = append(entries, entry) } } diff --git a/weed/filer2/stream.go b/weed/filer2/stream.go index 01b87cad1..0a1f943ea 100644 --- a/weed/filer2/stream.go +++ b/weed/filer2/stream.go @@ -27,7 +27,7 @@ func StreamContent(masterClient *wdclient.MasterClient, w io.Writer, chunks []*f for _, chunkView := range chunkViews { urlString := fileId2Url[chunkView.FileId] - _, err := util.ReadUrlAsStream(urlString, chunkView.Offset, int(chunkView.Size), func(data []byte) { + err := util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsFullChunk, chunkView.Offset, int(chunkView.Size), func(data []byte) { w.Write(data) }) if err != nil { diff --git a/weed/filesys/dirty_page.go b/weed/filesys/dirty_page.go index 67e1d57ef..ff3b8f885 100644 --- a/weed/filesys/dirty_page.go +++ b/weed/filesys/dirty_page.go @@ -174,7 +174,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(reader io.Reader, offset int64, } fileUrl := fmt.Sprintf("http://%s/%s", host, fileId) - uploadResult, err := operation.Upload(fileUrl, pages.f.Name, reader, false, "", nil, auth) + uploadResult, err := operation.Upload(fileUrl, pages.f.Name, pages.f.wfs.option.Cipher, reader, false, "", nil, auth) if err != nil { glog.V(0).Infof("upload data %v to %s: %v", pages.f.Name, fileUrl, err) return nil, fmt.Errorf("upload data: %v", err) @@ -185,11 +185,12 @@ func (pages *ContinuousDirtyPages) saveToStorage(reader io.Reader, offset int64, } return &filer_pb.FileChunk{ - FileId: fileId, - Offset: offset, - Size: uint64(size), - Mtime: time.Now().UnixNano(), - ETag: uploadResult.ETag, + FileId: fileId, + Offset: offset, + Size: uint64(size), + Mtime: time.Now().UnixNano(), + ETag: uploadResult.ETag, + CipherKey: uploadResult.CipherKey, }, nil } diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go index 83826fed5..77438b58e 100644 --- a/weed/filesys/wfs.go +++ b/weed/filesys/wfs.go @@ -39,8 +39,9 @@ type Option struct { MountCtime time.Time MountMtime time.Time - // whether the mount runs outside SeaweedFS containers - OutsideContainerClusterMode bool + OutsideContainerClusterMode bool // whether the mount runs outside SeaweedFS containers + Cipher bool // whether encrypt data on volume server + } var _ = fs.FS(&WFS{}) diff --git a/weed/operation/submit.go b/weed/operation/submit.go index 62f067430..8c7ed5d7b 100644 --- a/weed/operation/submit.go +++ b/weed/operation/submit.go @@ -189,7 +189,7 @@ func (fi FilePart) Upload(maxMB int, master string, jwt security.EncodedJwt, grp cm.DeleteChunks(master, grpcDialOption) } } else { - ret, e := Upload(fileUrl, baseName, fi.Reader, false, fi.MimeType, nil, jwt) + ret, e := Upload(fileUrl, baseName, false, fi.Reader, false, fi.MimeType, nil, jwt) if e != nil { return 0, e } @@ -202,8 +202,7 @@ func upload_one_chunk(filename string, reader io.Reader, master, fileUrl string, jwt security.EncodedJwt, ) (size uint32, e error) { glog.V(4).Info("Uploading part ", filename, " to ", fileUrl, "...") - uploadResult, uploadError := Upload(fileUrl, filename, reader, false, - "", nil, jwt) + uploadResult, uploadError := Upload(fileUrl, filename, false, reader, false, "", nil, jwt) if uploadError != nil { return 0, uploadError } @@ -221,6 +220,6 @@ func upload_chunked_file_manifest(fileUrl string, manifest *ChunkManifest, jwt s q := u.Query() q.Set("cm", "true") u.RawQuery = q.Encode() - _, e = Upload(u.String(), manifest.Name, bufReader, false, "application/json", nil, jwt) + _, e = Upload(u.String(), manifest.Name, false, bufReader, false, "application/json", nil, jwt) return e } diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index c387d0230..ba15aea78 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -22,10 +22,11 @@ import ( ) type UploadResult struct { - Name string `json:"name,omitempty"` - Size uint32 `json:"size,omitempty"` - Error string `json:"error,omitempty"` - ETag string `json:"eTag,omitempty"` + Name string `json:"name,omitempty"` + Size uint32 `json:"size,omitempty"` + Error string `json:"error,omitempty"` + ETag string `json:"eTag,omitempty"` + CipherKey []byte `json:"cipherKey,omitempty"` } var ( @@ -41,22 +42,22 @@ func init() { var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"") // Upload sends a POST request to a volume server to upload the content with adjustable compression level -func UploadWithLocalCompressionLevel(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt, compressionLevel int) (*UploadResult, error) { +func UploadWithLocalCompressionLevel(uploadUrl string, filename string, cipher bool, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt, compressionLevel int) (*UploadResult, error) { if compressionLevel < 1 { compressionLevel = 1 } if compressionLevel > 9 { compressionLevel = 9 } - return doUpload(uploadUrl, filename, reader, isGzipped, mtype, pairMap, compressionLevel, jwt) + return doUpload(uploadUrl, filename, cipher, reader, isGzipped, mtype, pairMap, compressionLevel, jwt) } // Upload sends a POST request to a volume server to upload the content with fast compression -func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) { - return doUpload(uploadUrl, filename, reader, isGzipped, mtype, pairMap, flate.BestSpeed, jwt) +func Upload(uploadUrl string, filename string, cipher bool, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) { + return doUpload(uploadUrl, filename, cipher, reader, isGzipped, mtype, pairMap, flate.BestSpeed, jwt) } -func doUpload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, compression int, jwt security.EncodedJwt) (*UploadResult, error) { +func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, compression int, jwt security.EncodedJwt) (*UploadResult, error) { contentIsGzipped := isGzipped shouldGzipNow := false if !isGzipped { @@ -65,7 +66,25 @@ func doUpload(uploadUrl string, filename string, reader io.Reader, isGzipped boo contentIsGzipped = true } } - return upload_content(uploadUrl, func(w io.Writer) (err error) { + // encrypt data + var cipherKey util.CipherKey + var clearDataLen int + if cipher { + clearData, err := ioutil.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("read raw input: %v", err) + } + clearDataLen = len(clearData) + cipherKey = util.GenCipherKey() + encryptedData, err := util.Encrypt(clearData, cipherKey) + if err != nil { + return nil, fmt.Errorf("encrypt input: %v", err) + } + reader = bytes.NewReader(encryptedData) + } + + // upload data + uploadResult, err := upload_content(uploadUrl, func(w io.Writer) (err error) { if shouldGzipNow { gzWriter, _ := gzip.NewWriterLevel(w, compression) _, err = io.Copy(gzWriter, reader) @@ -75,6 +94,14 @@ func doUpload(uploadUrl string, filename string, reader io.Reader, isGzipped boo } return }, filename, contentIsGzipped, mtype, pairMap, jwt) + + // remember cipher key + if uploadResult != nil && cipherKey != nil { + uploadResult.CipherKey = cipherKey + uploadResult.Size = uint32(clearDataLen) + } + + return uploadResult, err } func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) { diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto index 9ee552561..5983c84d8 100644 --- a/weed/pb/filer.proto +++ b/weed/pb/filer.proto @@ -99,6 +99,7 @@ message FileChunk { string source_file_id = 6; // to be deprecated FileId fid = 7; FileId source_fid = 8; + bytes cipher_key = 9; } message FileId { @@ -229,4 +230,5 @@ message GetFilerConfigurationResponse { uint32 max_mb = 4; string dir_buckets = 5; string dir_queues = 6; + bool cipher = 7; } diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go index 269abb8c7..92a12321c 100644 --- a/weed/pb/filer_pb/filer.pb.go +++ b/weed/pb/filer_pb/filer.pb.go @@ -287,6 +287,7 @@ type FileChunk struct { SourceFileId string `protobuf:"bytes,6,opt,name=source_file_id,json=sourceFileId" json:"source_file_id,omitempty"` Fid *FileId `protobuf:"bytes,7,opt,name=fid" json:"fid,omitempty"` SourceFid *FileId `protobuf:"bytes,8,opt,name=source_fid,json=sourceFid" json:"source_fid,omitempty"` + CipherKey []byte `protobuf:"bytes,9,opt,name=cipher_key,json=cipherKey,proto3" json:"cipher_key,omitempty"` } func (m *FileChunk) Reset() { *m = FileChunk{} } @@ -350,6 +351,13 @@ func (m *FileChunk) GetSourceFid() *FileId { return nil } +func (m *FileChunk) GetCipherKey() []byte { + if m != nil { + return m.CipherKey + } + return nil +} + type FileId struct { VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` FileKey uint64 `protobuf:"varint,2,opt,name=file_key,json=fileKey" json:"file_key,omitempty"` @@ -1014,6 +1022,7 @@ type GetFilerConfigurationResponse struct { MaxMb uint32 `protobuf:"varint,4,opt,name=max_mb,json=maxMb" json:"max_mb,omitempty"` DirBuckets string `protobuf:"bytes,5,opt,name=dir_buckets,json=dirBuckets" json:"dir_buckets,omitempty"` DirQueues string `protobuf:"bytes,6,opt,name=dir_queues,json=dirQueues" json:"dir_queues,omitempty"` + Cipher bool `protobuf:"varint,7,opt,name=cipher" json:"cipher,omitempty"` } func (m *GetFilerConfigurationResponse) Reset() { *m = GetFilerConfigurationResponse{} } @@ -1063,6 +1072,13 @@ func (m *GetFilerConfigurationResponse) GetDirQueues() string { return "" } +func (m *GetFilerConfigurationResponse) GetCipher() bool { + if m != nil { + return m.Cipher + } + return false +} + func init() { proto.RegisterType((*LookupDirectoryEntryRequest)(nil), "filer_pb.LookupDirectoryEntryRequest") proto.RegisterType((*LookupDirectoryEntryResponse)(nil), "filer_pb.LookupDirectoryEntryResponse") @@ -1594,113 +1610,114 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("filer.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 1713 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0xcb, 0x6e, 0xdb, 0xca, - 0x19, 0x36, 0x75, 0xe7, 0x2f, 0x29, 0xb1, 0x47, 0x76, 0xa2, 0xc8, 0x97, 0x3a, 0x74, 0x93, 0xba, - 0x48, 0xe0, 0x1a, 0x6e, 0x16, 0x49, 0xd3, 0x2e, 0x12, 0x5f, 0x0a, 0xa3, 0xce, 0xa5, 0x74, 0x52, - 0xa4, 0x28, 0x50, 0x82, 0x26, 0x47, 0xf2, 0xd4, 0x24, 0x47, 0x19, 0x0e, 0x6d, 0xa7, 0x8f, 0x52, - 0xa0, 0x8b, 0xbe, 0x47, 0xd1, 0x4d, 0x51, 0xa0, 0xeb, 0xf3, 0x08, 0xe7, 0x01, 0xce, 0xf2, 0xac, - 0x0f, 0x66, 0x86, 0xa4, 0x86, 0xa2, 0x6c, 0x27, 0xe7, 0x20, 0x3b, 0xce, 0x7f, 0x9b, 0x7f, 0xbe, - 0xff, 0x2a, 0x41, 0x7b, 0x48, 0x02, 0xcc, 0xb6, 0xc6, 0x8c, 0x72, 0x8a, 0x5a, 0xf2, 0xe0, 0x8c, - 0x4f, 0xac, 0x37, 0xb0, 0x7c, 0x44, 0xe9, 0x59, 0x32, 0xde, 0x23, 0x0c, 0x7b, 0x9c, 0xb2, 0x4f, - 0xfb, 0x11, 0x67, 0x9f, 0x6c, 0xfc, 0x31, 0xc1, 0x31, 0x47, 0x2b, 0x60, 0xfa, 0x19, 0xa3, 0x6f, - 0xac, 0x1b, 0x9b, 0xa6, 0x3d, 0x21, 0x20, 0x04, 0xb5, 0xc8, 0x0d, 0x71, 0xbf, 0x22, 0x19, 0xf2, - 0xdb, 0xda, 0x87, 0x95, 0xd9, 0x06, 0xe3, 0x31, 0x8d, 0x62, 0x8c, 0x1e, 0x40, 0x1d, 0x0b, 0x82, - 0xb4, 0xd6, 0xde, 0xb9, 0xbd, 0x95, 0xb9, 0xb2, 0xa5, 0xe4, 0x14, 0xd7, 0xfa, 0x8f, 0x01, 0xe8, - 0x88, 0xc4, 0x5c, 0x10, 0x09, 0x8e, 0x3f, 0xcf, 0x9f, 0x3b, 0xd0, 0x18, 0x33, 0x3c, 0x24, 0x97, - 0xa9, 0x47, 0xe9, 0x09, 0x3d, 0x86, 0x85, 0x98, 0xbb, 0x8c, 0x1f, 0x30, 0x1a, 0x1e, 0x90, 0x00, - 0xbf, 0x16, 0x4e, 0x57, 0xa5, 0x48, 0x99, 0x81, 0xb6, 0x00, 0x91, 0xc8, 0x0b, 0x92, 0x98, 0x9c, - 0xe3, 0xe3, 0x8c, 0xdb, 0xaf, 0xad, 0x1b, 0x9b, 0x2d, 0x7b, 0x06, 0x07, 0x2d, 0x42, 0x3d, 0x20, - 0x21, 0xe1, 0xfd, 0xfa, 0xba, 0xb1, 0xd9, 0xb5, 0xd5, 0xc1, 0xfa, 0x2d, 0xf4, 0x0a, 0xfe, 0x7f, - 0xd9, 0xf3, 0xff, 0x59, 0x81, 0xba, 0x24, 0xe4, 0x18, 0x1b, 0x13, 0x8c, 0xd1, 0x7d, 0xe8, 0x90, - 0xd8, 0x99, 0x00, 0x51, 0x91, 0xbe, 0xb5, 0x49, 0x9c, 0x63, 0x8e, 0x1e, 0x41, 0xc3, 0x3b, 0x4d, - 0xa2, 0xb3, 0xb8, 0x5f, 0x5d, 0xaf, 0x6e, 0xb6, 0x77, 0x7a, 0x93, 0x8b, 0xc4, 0x43, 0x77, 0x05, - 0xcf, 0x4e, 0x45, 0xd0, 0x53, 0x00, 0x97, 0x73, 0x46, 0x4e, 0x12, 0x8e, 0x63, 0xf9, 0xd2, 0xf6, - 0x4e, 0x5f, 0x53, 0x48, 0x62, 0xfc, 0x22, 0xe7, 0xdb, 0x9a, 0x2c, 0x7a, 0x06, 0x2d, 0x7c, 0xc9, - 0x71, 0xe4, 0x63, 0xbf, 0x5f, 0x97, 0x17, 0xad, 0x4e, 0xbd, 0x68, 0x6b, 0x3f, 0xe5, 0xab, 0xf7, - 0xe5, 0xe2, 0x83, 0xe7, 0xd0, 0x2d, 0xb0, 0xd0, 0x3c, 0x54, 0xcf, 0x70, 0x16, 0x55, 0xf1, 0x29, - 0x90, 0x3d, 0x77, 0x83, 0x44, 0x25, 0x58, 0xc7, 0x56, 0x87, 0xdf, 0x54, 0x9e, 0x1a, 0xd6, 0x1e, - 0x98, 0x07, 0x49, 0x10, 0xe4, 0x8a, 0x3e, 0x61, 0x99, 0xa2, 0x4f, 0xd8, 0x04, 0xe5, 0xca, 0xb5, - 0x28, 0xff, 0xdb, 0x80, 0x85, 0xfd, 0x73, 0x1c, 0xf1, 0xd7, 0x94, 0x93, 0x21, 0xf1, 0x5c, 0x4e, - 0x68, 0x84, 0x1e, 0x83, 0x49, 0x03, 0xdf, 0xb9, 0x36, 0x4c, 0x2d, 0x1a, 0xa4, 0x5e, 0x3f, 0x06, - 0x33, 0xc2, 0x17, 0xce, 0xb5, 0xd7, 0xb5, 0x22, 0x7c, 0xa1, 0xa4, 0x37, 0xa0, 0xeb, 0xe3, 0x00, - 0x73, 0xec, 0xe4, 0xd1, 0x11, 0xa1, 0xeb, 0x28, 0xe2, 0xae, 0x0a, 0xc7, 0x43, 0xb8, 0x2d, 0x4c, - 0x8e, 0x5d, 0x86, 0x23, 0xee, 0x8c, 0x5d, 0x7e, 0x2a, 0x63, 0x62, 0xda, 0xdd, 0x08, 0x5f, 0xbc, - 0x95, 0xd4, 0xb7, 0x2e, 0x3f, 0xb5, 0xbe, 0x37, 0xc0, 0xcc, 0x83, 0x89, 0xee, 0x42, 0x53, 0x5c, - 0xeb, 0x10, 0x3f, 0x45, 0xa2, 0x21, 0x8e, 0x87, 0xbe, 0xa8, 0x0a, 0x3a, 0x1c, 0xc6, 0x98, 0x4b, - 0xf7, 0xaa, 0x76, 0x7a, 0x12, 0x99, 0x15, 0x93, 0xbf, 0xab, 0x42, 0xa8, 0xd9, 0xf2, 0x5b, 0x20, - 0x1e, 0x72, 0x12, 0x62, 0x79, 0x61, 0xd5, 0x56, 0x07, 0xd4, 0x83, 0x3a, 0x76, 0xb8, 0x3b, 0x92, - 0x19, 0x6e, 0xda, 0x35, 0xfc, 0xce, 0x1d, 0xa1, 0x9f, 0xc3, 0xad, 0x98, 0x26, 0xcc, 0xc3, 0x4e, - 0x76, 0x6d, 0x43, 0x72, 0x3b, 0x8a, 0x7a, 0xa0, 0x2e, 0xb7, 0xa0, 0x3a, 0x24, 0x7e, 0xbf, 0x29, - 0x81, 0x99, 0x2f, 0x26, 0xe1, 0xa1, 0x6f, 0x0b, 0x26, 0xfa, 0x15, 0x40, 0x6e, 0xc9, 0xef, 0xb7, - 0xae, 0x10, 0x35, 0x33, 0xbb, 0xbe, 0xf5, 0x01, 0x1a, 0xa9, 0xf9, 0x65, 0x30, 0xcf, 0x69, 0x90, - 0x84, 0xf9, 0xb3, 0xbb, 0x76, 0x4b, 0x11, 0x0e, 0x7d, 0x74, 0x0f, 0x64, 0x9f, 0x73, 0x44, 0x56, - 0x55, 0xe4, 0x23, 0x25, 0x42, 0x7f, 0xc0, 0xb2, 0x53, 0x78, 0x94, 0x9e, 0x11, 0xf5, 0xfa, 0xa6, - 0x9d, 0x9e, 0xac, 0xef, 0x2a, 0x70, 0xab, 0x98, 0xee, 0xe2, 0x0a, 0x69, 0x45, 0x62, 0x65, 0x48, - 0x33, 0xd2, 0xec, 0x71, 0x01, 0xaf, 0x8a, 0x8e, 0x57, 0xa6, 0x12, 0x52, 0x5f, 0x5d, 0xd0, 0x55, - 0x2a, 0xaf, 0xa8, 0x8f, 0x45, 0xb6, 0x26, 0xc4, 0x97, 0x00, 0x77, 0x6d, 0xf1, 0x29, 0x28, 0x23, - 0xe2, 0xa7, 0xed, 0x43, 0x7c, 0x4a, 0xf7, 0x98, 0xb4, 0xdb, 0x50, 0x21, 0x53, 0x27, 0x11, 0xb2, - 0x50, 0x50, 0x9b, 0x2a, 0x0e, 0xe2, 0x1b, 0xad, 0x43, 0x9b, 0xe1, 0x71, 0x90, 0x66, 0xaf, 0x84, - 0xcf, 0xb4, 0x75, 0x12, 0x5a, 0x03, 0xf0, 0x68, 0x10, 0x60, 0x4f, 0x0a, 0x98, 0x52, 0x40, 0xa3, - 0x88, 0xcc, 0xe1, 0x3c, 0x70, 0x62, 0xec, 0xf5, 0x61, 0xdd, 0xd8, 0xac, 0xdb, 0x0d, 0xce, 0x83, - 0x63, 0xec, 0x89, 0x77, 0x24, 0x31, 0x66, 0x8e, 0x6c, 0x40, 0x6d, 0xa9, 0xd7, 0x12, 0x04, 0xd9, - 0x26, 0x57, 0x01, 0x46, 0x8c, 0x26, 0x63, 0xc5, 0xed, 0xac, 0x57, 0x45, 0x2f, 0x96, 0x14, 0xc9, - 0x7e, 0x00, 0xb7, 0xe2, 0x4f, 0x61, 0x40, 0xa2, 0x33, 0x87, 0xbb, 0x6c, 0x84, 0x79, 0xbf, 0xab, - 0x72, 0x38, 0xa5, 0xbe, 0x93, 0x44, 0x6b, 0x0c, 0x68, 0x97, 0x61, 0x97, 0xe3, 0x2f, 0x18, 0x3b, - 0x9f, 0x57, 0xdd, 0x68, 0x09, 0x1a, 0xd4, 0xc1, 0x97, 0x5e, 0x90, 0x16, 0x59, 0x9d, 0xee, 0x5f, - 0x7a, 0x81, 0xf5, 0x08, 0x7a, 0x85, 0x1b, 0xd3, 0xc6, 0xbc, 0x08, 0x75, 0xcc, 0x18, 0xcd, 0xda, - 0x88, 0x3a, 0x58, 0x7f, 0x06, 0xf4, 0x7e, 0xec, 0x7f, 0x0d, 0xf7, 0xac, 0x25, 0xe8, 0x15, 0x4c, - 0x2b, 0x3f, 0xac, 0xff, 0x19, 0x80, 0xf6, 0x64, 0x37, 0xf8, 0x69, 0x83, 0x58, 0xd4, 0xa7, 0x18, - 0x12, 0xaa, 0xdb, 0xf8, 0x2e, 0x77, 0xd3, 0x11, 0xd6, 0x21, 0xb1, 0xb2, 0xbf, 0xe7, 0x72, 0x37, - 0x1d, 0x25, 0x0c, 0x7b, 0x09, 0x13, 0x53, 0x4d, 0x26, 0xa1, 0x1c, 0x25, 0x76, 0x46, 0x42, 0x4f, - 0xe0, 0x0e, 0x19, 0x45, 0x94, 0xe1, 0x89, 0x98, 0xa3, 0xa0, 0x6a, 0x48, 0xe1, 0x45, 0xc5, 0xcd, - 0x15, 0xf6, 0x25, 0x72, 0x8f, 0xa0, 0x57, 0x78, 0xc6, 0xb5, 0x30, 0xff, 0xc3, 0x80, 0xfe, 0x0b, - 0x4e, 0x43, 0xe2, 0xd9, 0x58, 0x38, 0x5f, 0x78, 0xfa, 0x06, 0x74, 0x45, 0x3f, 0x9e, 0x7e, 0x7e, - 0x87, 0x06, 0xfe, 0x64, 0xde, 0xdd, 0x03, 0xd1, 0x92, 0x1d, 0x0d, 0x85, 0x26, 0x0d, 0x7c, 0x99, - 0x89, 0x1b, 0x20, 0xfa, 0xa6, 0xa6, 0xaf, 0x26, 0x7f, 0x27, 0xc2, 0x17, 0x05, 0x7d, 0x21, 0x24, - 0xf5, 0x55, 0xb3, 0x6d, 0x46, 0xf8, 0x42, 0xe8, 0x5b, 0xcb, 0x70, 0x6f, 0x86, 0x6f, 0x69, 0xb8, - 0xfe, 0x6f, 0x40, 0xef, 0x45, 0x1c, 0x93, 0x51, 0xf4, 0x27, 0xd9, 0x76, 0x32, 0xa7, 0x17, 0xa1, - 0xee, 0xd1, 0x24, 0xe2, 0xd2, 0xd9, 0xba, 0xad, 0x0e, 0x53, 0x95, 0x58, 0x29, 0x55, 0xe2, 0x54, - 0x2d, 0x57, 0xcb, 0xb5, 0xac, 0xd5, 0x6a, 0xad, 0x50, 0xab, 0x3f, 0x83, 0xb6, 0x08, 0xb2, 0xe3, - 0xe1, 0x88, 0x63, 0x96, 0x76, 0x6a, 0x10, 0xa4, 0x5d, 0x49, 0x11, 0x02, 0xfa, 0x44, 0x51, 0xcd, - 0x1a, 0xc6, 0x93, 0x71, 0xf2, 0xad, 0x01, 0x8b, 0xc5, 0xa7, 0xa4, 0x31, 0xbb, 0x72, 0xb2, 0x88, - 0x56, 0xc6, 0x82, 0xf4, 0x1d, 0xe2, 0x53, 0x34, 0x85, 0x71, 0x72, 0x12, 0x10, 0xcf, 0x11, 0x0c, - 0xe5, 0xbf, 0xa9, 0x28, 0xef, 0x59, 0x30, 0x41, 0xa5, 0xa6, 0xa3, 0x82, 0xa0, 0xe6, 0x26, 0xfc, - 0x34, 0x9b, 0x2e, 0xe2, 0x7b, 0x0a, 0xa9, 0xc6, 0x4d, 0x48, 0x35, 0xcb, 0x48, 0xe5, 0x99, 0xd6, - 0xd2, 0x33, 0xed, 0x09, 0xf4, 0xd4, 0x7a, 0x5a, 0x0c, 0xd7, 0x2a, 0x40, 0x3e, 0x47, 0xe2, 0xbe, - 0xa1, 0x9a, 0x59, 0x36, 0x48, 0x62, 0xeb, 0x77, 0x60, 0x1e, 0x51, 0x65, 0x37, 0x46, 0xdb, 0x60, - 0x06, 0xd9, 0x41, 0x8a, 0xb6, 0x77, 0xd0, 0xa4, 0xc6, 0x33, 0x39, 0x7b, 0x22, 0x64, 0x3d, 0x87, - 0x56, 0x46, 0xce, 0x30, 0x33, 0xae, 0xc2, 0xac, 0x32, 0x85, 0x99, 0xf5, 0x5f, 0x03, 0x16, 0x8b, - 0x2e, 0xa7, 0x61, 0x79, 0x0f, 0xdd, 0xfc, 0x0a, 0x27, 0x74, 0xc7, 0xa9, 0x2f, 0xdb, 0xba, 0x2f, - 0x65, 0xb5, 0xdc, 0xc1, 0xf8, 0x95, 0x3b, 0x56, 0xb9, 0xdc, 0x09, 0x34, 0xd2, 0xe0, 0x1d, 0x2c, - 0x94, 0x44, 0x66, 0xec, 0x66, 0xbf, 0xd4, 0x77, 0xb3, 0xc2, 0x7e, 0x99, 0x6b, 0xeb, 0x0b, 0xdb, - 0x33, 0xb8, 0xab, 0xda, 0xc1, 0x6e, 0x1e, 0xc3, 0x0c, 0xfb, 0x62, 0xa8, 0x8d, 0xe9, 0x50, 0x5b, - 0x03, 0xe8, 0x97, 0x55, 0xd3, 0xf2, 0x1b, 0xc1, 0xc2, 0x31, 0x77, 0x39, 0x89, 0x39, 0xf1, 0xf2, - 0x1f, 0x09, 0x53, 0xb9, 0x61, 0xdc, 0x34, 0x11, 0xcb, 0x75, 0x38, 0x0f, 0x55, 0xce, 0xb3, 0xfc, - 0x15, 0x9f, 0x22, 0x0a, 0x48, 0xbf, 0x29, 0x8d, 0xc1, 0x57, 0xb8, 0x4a, 0xe4, 0x03, 0xa7, 0xdc, - 0x0d, 0xd4, 0xc6, 0x51, 0x93, 0x1b, 0x87, 0x29, 0x29, 0x72, 0xe5, 0x50, 0x43, 0xd9, 0x57, 0xdc, - 0xba, 0xda, 0x47, 0x04, 0x41, 0x32, 0x57, 0x01, 0x64, 0xa9, 0xaa, 0x2a, 0x6b, 0x28, 0x5d, 0x41, - 0xd9, 0x15, 0x04, 0x6b, 0x0d, 0x56, 0x7e, 0x8f, 0xb9, 0xd8, 0x9d, 0xd8, 0x2e, 0x8d, 0x86, 0x64, - 0x94, 0x30, 0x57, 0x0b, 0x85, 0xf5, 0x8d, 0x01, 0xab, 0x57, 0x08, 0xa4, 0x0f, 0xee, 0x43, 0x33, - 0x74, 0x63, 0x8e, 0x59, 0x56, 0x25, 0xd9, 0x71, 0x1a, 0x8a, 0xca, 0x4d, 0x50, 0x54, 0x4b, 0x50, - 0x2c, 0x41, 0x23, 0x74, 0x2f, 0x9d, 0xf0, 0x24, 0x5d, 0x8e, 0xea, 0xa1, 0x7b, 0xf9, 0xea, 0x44, - 0x76, 0x36, 0xc2, 0x9c, 0x93, 0xc4, 0x3b, 0xc3, 0x3c, 0xce, 0x3b, 0x1b, 0x61, 0x2f, 0x15, 0x45, - 0x3c, 0x5a, 0x08, 0x7c, 0x4c, 0x70, 0x82, 0xe3, 0xb4, 0x57, 0x88, 0xe1, 0xf8, 0x47, 0x49, 0xd8, - 0xf9, 0x57, 0x0b, 0x3a, 0xc7, 0xd8, 0xbd, 0xc0, 0xd8, 0x97, 0x0f, 0x43, 0xa3, 0xac, 0xa0, 0x8a, - 0x3f, 0x51, 0xd1, 0x83, 0xe9, 0xca, 0x99, 0xf9, 0x9b, 0x78, 0xf0, 0xf0, 0x26, 0xb1, 0x34, 0x37, - 0xe7, 0xd0, 0x6b, 0x68, 0x6b, 0xbf, 0x01, 0xd1, 0x8a, 0xa6, 0x58, 0xfa, 0x69, 0x3b, 0x58, 0xbd, - 0x82, 0x9b, 0x59, 0xdb, 0x36, 0xd0, 0x11, 0xb4, 0xb5, 0xd5, 0x45, 0xb7, 0x57, 0xde, 0xa1, 0x74, - 0x7b, 0x33, 0xf6, 0x1d, 0x6b, 0x4e, 0x58, 0xd3, 0x16, 0x10, 0xdd, 0x5a, 0x79, 0xe5, 0xd1, 0xad, - 0xcd, 0xda, 0x5a, 0xa4, 0x35, 0x6d, 0xde, 0xeb, 0xd6, 0xca, 0xdb, 0x8c, 0x6e, 0x6d, 0xc6, 0x92, - 0x60, 0xcd, 0xa1, 0x0f, 0xd0, 0x3b, 0xe6, 0x0c, 0xbb, 0xe1, 0x84, 0x3d, 0x85, 0xe0, 0x8f, 0xb0, - 0xba, 0x69, 0x6c, 0x1b, 0xe8, 0xaf, 0xb0, 0x50, 0x9a, 0xe6, 0xc8, 0x9a, 0x68, 0x5e, 0xb5, 0x86, - 0x0c, 0x36, 0xae, 0x95, 0xc9, 0x3d, 0x7f, 0x03, 0x1d, 0x7d, 0x88, 0x22, 0xcd, 0xa9, 0x19, 0x7b, - 0xc2, 0x60, 0xed, 0x2a, 0xb6, 0x6e, 0x50, 0xef, 0xe3, 0xba, 0xc1, 0x19, 0x93, 0x4c, 0x37, 0x38, - 0xab, 0xfd, 0x5b, 0x73, 0xe8, 0x2f, 0x30, 0x3f, 0xdd, 0x4f, 0xd1, 0xfd, 0x69, 0xe8, 0x4a, 0x6d, - 0x7a, 0x60, 0x5d, 0x27, 0x92, 0x1b, 0x3f, 0x04, 0x98, 0xb4, 0x49, 0xb4, 0x3c, 0xd1, 0x29, 0xb5, - 0xe9, 0xc1, 0xca, 0x6c, 0x66, 0x6e, 0xea, 0x6f, 0xb0, 0x34, 0xb3, 0x17, 0x21, 0xad, 0x00, 0xaf, - 0xeb, 0x66, 0x83, 0x5f, 0xdc, 0x28, 0x97, 0xdd, 0xf5, 0x72, 0x0d, 0xe6, 0x63, 0xd5, 0x22, 0x86, - 0xf1, 0x96, 0x17, 0x10, 0x1c, 0xf1, 0x97, 0x20, 0x35, 0xde, 0x32, 0xca, 0xe9, 0x49, 0x43, 0xfe, - 0x6f, 0xf6, 0xeb, 0x1f, 0x02, 0x00, 0x00, 0xff, 0xff, 0xdb, 0x25, 0xa7, 0x6f, 0x46, 0x13, 0x00, - 0x00, + // 1742 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0x4f, 0x6f, 0xdc, 0xc6, + 0x15, 0x37, 0xf7, 0x3f, 0xdf, 0xee, 0x3a, 0xd2, 0xac, 0x9c, 0xac, 0xd7, 0x92, 0xab, 0xd0, 0x75, + 0xaa, 0xc2, 0x86, 0x6a, 0xa8, 0x39, 0x24, 0x4d, 0x7b, 0xb0, 0x65, 0xb9, 0x30, 0x62, 0x3b, 0x2e, + 0x65, 0x17, 0x29, 0x0a, 0x94, 0xa0, 0xc8, 0xd9, 0xd5, 0x54, 0x24, 0x87, 0x19, 0x0e, 0x2d, 0xb9, + 0xdf, 0xa3, 0x97, 0x02, 0x3d, 0xf4, 0x7b, 0x14, 0xbd, 0x14, 0x05, 0xfa, 0x39, 0x7a, 0xec, 0xa1, + 0x9f, 0xa1, 0x98, 0x37, 0x24, 0x77, 0xb8, 0x5c, 0x49, 0x49, 0x8b, 0xdc, 0x38, 0xef, 0xdf, 0xbc, + 0xf9, 0xbd, 0xbf, 0xbb, 0x30, 0x9c, 0xb3, 0x88, 0x8a, 0xfd, 0x54, 0x70, 0xc9, 0xc9, 0x00, 0x0f, + 0x5e, 0x7a, 0xe2, 0x7c, 0x05, 0x77, 0x5e, 0x70, 0x7e, 0x96, 0xa7, 0x4f, 0x99, 0xa0, 0x81, 0xe4, + 0xe2, 0xfd, 0x51, 0x22, 0xc5, 0x7b, 0x97, 0x7e, 0x93, 0xd3, 0x4c, 0x92, 0x6d, 0xb0, 0xc3, 0x92, + 0x31, 0xb5, 0x76, 0xad, 0x3d, 0xdb, 0x5d, 0x12, 0x08, 0x81, 0x4e, 0xe2, 0xc7, 0x74, 0xda, 0x42, + 0x06, 0x7e, 0x3b, 0x47, 0xb0, 0xbd, 0xde, 0x60, 0x96, 0xf2, 0x24, 0xa3, 0xe4, 0x3e, 0x74, 0xa9, + 0x22, 0xa0, 0xb5, 0xe1, 0xc1, 0x07, 0xfb, 0xa5, 0x2b, 0xfb, 0x5a, 0x4e, 0x73, 0x9d, 0xbf, 0x59, + 0x40, 0x5e, 0xb0, 0x4c, 0x2a, 0x22, 0xa3, 0xd9, 0xb7, 0xf3, 0xe7, 0x43, 0xe8, 0xa5, 0x82, 0xce, + 0xd9, 0x45, 0xe1, 0x51, 0x71, 0x22, 0x0f, 0x61, 0x33, 0x93, 0xbe, 0x90, 0xcf, 0x04, 0x8f, 0x9f, + 0xb1, 0x88, 0xbe, 0x52, 0x4e, 0xb7, 0x51, 0xa4, 0xc9, 0x20, 0xfb, 0x40, 0x58, 0x12, 0x44, 0x79, + 0xc6, 0xde, 0xd1, 0xe3, 0x92, 0x3b, 0xed, 0xec, 0x5a, 0x7b, 0x03, 0x77, 0x0d, 0x87, 0x6c, 0x41, + 0x37, 0x62, 0x31, 0x93, 0xd3, 0xee, 0xae, 0xb5, 0x37, 0x76, 0xf5, 0xc1, 0xf9, 0x39, 0x4c, 0x6a, + 0xfe, 0x7f, 0xb7, 0xe7, 0xff, 0xb9, 0x05, 0x5d, 0x24, 0x54, 0x18, 0x5b, 0x4b, 0x8c, 0xc9, 0xc7, + 0x30, 0x62, 0x99, 0xb7, 0x04, 0xa2, 0x85, 0xbe, 0x0d, 0x59, 0x56, 0x61, 0x4e, 0x1e, 0x40, 0x2f, + 0x38, 0xcd, 0x93, 0xb3, 0x6c, 0xda, 0xde, 0x6d, 0xef, 0x0d, 0x0f, 0x26, 0xcb, 0x8b, 0xd4, 0x43, + 0x0f, 0x15, 0xcf, 0x2d, 0x44, 0xc8, 0x67, 0x00, 0xbe, 0x94, 0x82, 0x9d, 0xe4, 0x92, 0x66, 0xf8, + 0xd2, 0xe1, 0xc1, 0xd4, 0x50, 0xc8, 0x33, 0xfa, 0xb8, 0xe2, 0xbb, 0x86, 0x2c, 0xf9, 0x1c, 0x06, + 0xf4, 0x42, 0xd2, 0x24, 0xa4, 0xe1, 0xb4, 0x8b, 0x17, 0xed, 0xac, 0xbc, 0x68, 0xff, 0xa8, 0xe0, + 0xeb, 0xf7, 0x55, 0xe2, 0xb3, 0x2f, 0x60, 0x5c, 0x63, 0x91, 0x0d, 0x68, 0x9f, 0xd1, 0x32, 0xaa, + 0xea, 0x53, 0x21, 0xfb, 0xce, 0x8f, 0x72, 0x9d, 0x60, 0x23, 0x57, 0x1f, 0x7e, 0xd6, 0xfa, 0xcc, + 0x72, 0x9e, 0x82, 0xfd, 0x2c, 0x8f, 0xa2, 0x4a, 0x31, 0x64, 0xa2, 0x54, 0x0c, 0x99, 0x58, 0xa2, + 0xdc, 0xba, 0x12, 0xe5, 0xbf, 0x5a, 0xb0, 0x79, 0xf4, 0x8e, 0x26, 0xf2, 0x15, 0x97, 0x6c, 0xce, + 0x02, 0x5f, 0x32, 0x9e, 0x90, 0x87, 0x60, 0xf3, 0x28, 0xf4, 0xae, 0x0c, 0xd3, 0x80, 0x47, 0x85, + 0xd7, 0x0f, 0xc1, 0x4e, 0xe8, 0xb9, 0x77, 0xe5, 0x75, 0x83, 0x84, 0x9e, 0x6b, 0xe9, 0x7b, 0x30, + 0x0e, 0x69, 0x44, 0x25, 0xf5, 0xaa, 0xe8, 0xa8, 0xd0, 0x8d, 0x34, 0xf1, 0x50, 0x87, 0xe3, 0x13, + 0xf8, 0x40, 0x99, 0x4c, 0x7d, 0x41, 0x13, 0xe9, 0xa5, 0xbe, 0x3c, 0xc5, 0x98, 0xd8, 0xee, 0x38, + 0xa1, 0xe7, 0xaf, 0x91, 0xfa, 0xda, 0x97, 0xa7, 0xce, 0x1f, 0x5b, 0x60, 0x57, 0xc1, 0x24, 0x1f, + 0x41, 0x5f, 0x5d, 0xeb, 0xb1, 0xb0, 0x40, 0xa2, 0xa7, 0x8e, 0xcf, 0x43, 0x55, 0x15, 0x7c, 0x3e, + 0xcf, 0xa8, 0x44, 0xf7, 0xda, 0x6e, 0x71, 0x52, 0x99, 0x95, 0xb1, 0x3f, 0xe8, 0x42, 0xe8, 0xb8, + 0xf8, 0xad, 0x10, 0x8f, 0x25, 0x8b, 0x29, 0x5e, 0xd8, 0x76, 0xf5, 0x81, 0x4c, 0xa0, 0x4b, 0x3d, + 0xe9, 0x2f, 0x30, 0xc3, 0x6d, 0xb7, 0x43, 0xdf, 0xf8, 0x0b, 0xf2, 0x43, 0xb8, 0x99, 0xf1, 0x5c, + 0x04, 0xd4, 0x2b, 0xaf, 0xed, 0x21, 0x77, 0xa4, 0xa9, 0xcf, 0xf4, 0xe5, 0x0e, 0xb4, 0xe7, 0x2c, + 0x9c, 0xf6, 0x11, 0x98, 0x8d, 0x7a, 0x12, 0x3e, 0x0f, 0x5d, 0xc5, 0x24, 0x3f, 0x01, 0xa8, 0x2c, + 0x85, 0xd3, 0xc1, 0x25, 0xa2, 0x76, 0x69, 0x37, 0x24, 0x3b, 0x00, 0x01, 0x4b, 0x4f, 0xa9, 0xf0, + 0x54, 0xc2, 0xd8, 0x98, 0x1c, 0xb6, 0xa6, 0x7c, 0x49, 0xdf, 0x3b, 0x5f, 0x43, 0xaf, 0xb8, 0xfd, + 0x0e, 0xd8, 0xef, 0x78, 0x94, 0xc7, 0x15, 0x2a, 0x63, 0x77, 0xa0, 0x09, 0xcf, 0x43, 0x72, 0x1b, + 0xb0, 0x0d, 0xa2, 0x8d, 0x16, 0x62, 0x80, 0x00, 0x7e, 0x49, 0xb1, 0x91, 0x04, 0x9c, 0x9f, 0x31, + 0x0d, 0x4e, 0xdf, 0x2d, 0x4e, 0xce, 0x7f, 0x5a, 0x70, 0xb3, 0x5e, 0x0d, 0xea, 0x0a, 0xb4, 0x82, + 0x50, 0x5a, 0x68, 0x06, 0xcd, 0x1e, 0xd7, 0xe0, 0x6c, 0x99, 0x70, 0x96, 0x2a, 0x31, 0x0f, 0xf5, + 0x05, 0x63, 0xad, 0xf2, 0x92, 0x87, 0x54, 0x25, 0x73, 0xce, 0x42, 0xc4, 0x7f, 0xec, 0xaa, 0x4f, + 0x45, 0x59, 0xb0, 0xb0, 0xe8, 0x2e, 0xea, 0x13, 0xdd, 0x13, 0x68, 0xb7, 0xa7, 0x23, 0xaa, 0x4f, + 0x2a, 0xa2, 0xb1, 0xa2, 0xf6, 0x75, 0x98, 0xd4, 0x37, 0xd9, 0x85, 0xa1, 0xa0, 0x69, 0x54, 0x24, + 0x37, 0xa2, 0x6b, 0xbb, 0x26, 0x89, 0xdc, 0x05, 0x08, 0x78, 0x14, 0xd1, 0x00, 0x05, 0x6c, 0x14, + 0x30, 0x28, 0x2a, 0xb1, 0xa4, 0x8c, 0xbc, 0x8c, 0x06, 0x53, 0xd8, 0xb5, 0xf6, 0xba, 0x6e, 0x4f, + 0xca, 0xe8, 0x98, 0x06, 0xea, 0x1d, 0x79, 0x46, 0x85, 0x87, 0xfd, 0x69, 0x88, 0x7a, 0x03, 0x45, + 0xc0, 0x2e, 0xba, 0x03, 0xb0, 0x10, 0x3c, 0x4f, 0x35, 0x77, 0xb4, 0xdb, 0x56, 0xad, 0x1a, 0x29, + 0xc8, 0xbe, 0x0f, 0x37, 0xb3, 0xf7, 0x71, 0xc4, 0x92, 0x33, 0x4f, 0xfa, 0x62, 0x41, 0xe5, 0x74, + 0xac, 0x53, 0xbc, 0xa0, 0xbe, 0x41, 0xa2, 0x93, 0x02, 0x39, 0x14, 0xd4, 0x97, 0xf4, 0x3b, 0x4c, + 0xa5, 0x6f, 0x57, 0xfc, 0xe4, 0x16, 0xf4, 0xb8, 0x47, 0x2f, 0x82, 0xa8, 0xa8, 0xc1, 0x2e, 0x3f, + 0xba, 0x08, 0x22, 0xe7, 0x01, 0x4c, 0x6a, 0x37, 0x16, 0x7d, 0x7b, 0x0b, 0xba, 0x54, 0x08, 0x5e, + 0x76, 0x19, 0x7d, 0x70, 0x7e, 0x03, 0xe4, 0x6d, 0x1a, 0x7e, 0x1f, 0xee, 0x39, 0xb7, 0x60, 0x52, + 0x33, 0xad, 0xfd, 0x70, 0xfe, 0x61, 0x01, 0x79, 0x8a, 0xcd, 0xe2, 0xff, 0x9b, 0xd3, 0xaa, 0x7c, + 0xd5, 0x0c, 0xd1, 0xcd, 0x28, 0xf4, 0xa5, 0x5f, 0x4c, 0xb8, 0x11, 0xcb, 0xb4, 0xfd, 0xa7, 0xbe, + 0xf4, 0x8b, 0x49, 0x23, 0x68, 0x90, 0x0b, 0x35, 0xf4, 0x30, 0x09, 0x71, 0xd2, 0xb8, 0x25, 0x89, + 0x7c, 0x0a, 0x1f, 0xb2, 0x45, 0xc2, 0x05, 0x5d, 0x8a, 0x79, 0x1a, 0xaa, 0x1e, 0x0a, 0x6f, 0x69, + 0x6e, 0xa5, 0x70, 0x84, 0xc8, 0x3d, 0x80, 0x49, 0xed, 0x19, 0x57, 0xc2, 0xfc, 0x27, 0x0b, 0xa6, + 0x8f, 0x25, 0x8f, 0x59, 0xe0, 0x52, 0xe5, 0x7c, 0xed, 0xe9, 0xf7, 0x60, 0xac, 0xda, 0xf5, 0xea, + 0xf3, 0x47, 0x3c, 0x0a, 0x97, 0xe3, 0xf0, 0x36, 0xa8, 0x8e, 0xed, 0x19, 0x28, 0xf4, 0x79, 0x14, + 0x62, 0x26, 0xde, 0x03, 0xd5, 0x56, 0x0d, 0x7d, 0xbd, 0x18, 0x8c, 0x12, 0x7a, 0x5e, 0xd3, 0x57, + 0x42, 0xa8, 0xaf, 0x7b, 0x71, 0x3f, 0xa1, 0xe7, 0x4a, 0xdf, 0xb9, 0x03, 0xb7, 0xd7, 0xf8, 0x56, + 0x84, 0xeb, 0x9f, 0x16, 0x4c, 0x1e, 0x67, 0x19, 0x5b, 0x24, 0xbf, 0xc6, 0xb6, 0x53, 0x3a, 0xbd, + 0x05, 0xdd, 0x80, 0xe7, 0x89, 0x44, 0x67, 0xbb, 0xae, 0x3e, 0xac, 0x54, 0x62, 0xab, 0x51, 0x89, + 0x2b, 0xb5, 0xdc, 0x6e, 0xd6, 0xb2, 0x51, 0xab, 0x9d, 0x5a, 0xad, 0xfe, 0x00, 0x86, 0x2a, 0xc8, + 0x5e, 0x40, 0x13, 0x49, 0x45, 0xd1, 0xc8, 0x41, 0x91, 0x0e, 0x91, 0xa2, 0x04, 0xcc, 0x81, 0xa3, + 0x7b, 0x39, 0xa4, 0xcb, 0x69, 0xf3, 0x2f, 0x0b, 0xb6, 0xea, 0x4f, 0x29, 0x62, 0x76, 0xe9, 0xe0, + 0x51, 0xad, 0x4c, 0x44, 0xc5, 0x3b, 0xd4, 0xa7, 0x6a, 0x0a, 0x69, 0x7e, 0x12, 0xb1, 0xc0, 0x53, + 0x0c, 0xed, 0xbf, 0xad, 0x29, 0x6f, 0x45, 0xb4, 0x44, 0xa5, 0x63, 0xa2, 0x42, 0xa0, 0xe3, 0xe7, + 0xf2, 0xb4, 0x1c, 0x3e, 0xea, 0x7b, 0x05, 0xa9, 0xde, 0x75, 0x48, 0xf5, 0x9b, 0x48, 0x55, 0x99, + 0x36, 0x30, 0x33, 0xed, 0x53, 0x98, 0xe8, 0xed, 0xb5, 0x1e, 0xae, 0x1d, 0x80, 0x6a, 0x8e, 0x64, + 0x53, 0x4b, 0x37, 0xb3, 0x72, 0x90, 0x64, 0xce, 0x2f, 0xc0, 0x7e, 0xc1, 0xb5, 0xdd, 0x8c, 0x3c, + 0x02, 0x3b, 0x2a, 0x0f, 0x28, 0x3a, 0x3c, 0x20, 0xcb, 0x1a, 0x2f, 0xe5, 0xdc, 0xa5, 0x90, 0xf3, + 0x05, 0x0c, 0x4a, 0x72, 0x89, 0x99, 0x75, 0x19, 0x66, 0xad, 0x15, 0xcc, 0x9c, 0xbf, 0x5b, 0xb0, + 0x55, 0x77, 0xb9, 0x08, 0xcb, 0x5b, 0x18, 0x57, 0x57, 0x78, 0xb1, 0x9f, 0x16, 0xbe, 0x3c, 0x32, + 0x7d, 0x69, 0xaa, 0x55, 0x0e, 0x66, 0x2f, 0xfd, 0x54, 0xe7, 0xf2, 0x28, 0x32, 0x48, 0xb3, 0x37, + 0xb0, 0xd9, 0x10, 0x59, 0xb3, 0xba, 0xfd, 0xd8, 0x5c, 0xdd, 0x6a, 0xeb, 0x67, 0xa5, 0x6d, 0xee, + 0x73, 0x9f, 0xc3, 0x47, 0xba, 0x1d, 0x1c, 0x56, 0x31, 0x2c, 0xb1, 0xaf, 0x87, 0xda, 0x5a, 0x0d, + 0xb5, 0x33, 0x83, 0x69, 0x53, 0xb5, 0x28, 0xbf, 0x05, 0x6c, 0x1e, 0x4b, 0x5f, 0xb2, 0x4c, 0xb2, + 0xa0, 0xfa, 0x0d, 0xb1, 0x92, 0x1b, 0xd6, 0x75, 0x13, 0xb1, 0x59, 0x87, 0x1b, 0xd0, 0x96, 0xb2, + 0xcc, 0x5f, 0xf5, 0xa9, 0xa2, 0x40, 0xcc, 0x9b, 0x8a, 0x18, 0x7c, 0x0f, 0x57, 0xa9, 0x7c, 0x90, + 0x5c, 0xfa, 0x91, 0xde, 0x38, 0x3a, 0xb8, 0x71, 0xd8, 0x48, 0xc1, 0x95, 0x43, 0x0f, 0xe5, 0x50, + 0x73, 0xbb, 0x7a, 0x1f, 0x51, 0x04, 0x64, 0xee, 0x00, 0x60, 0xa9, 0xea, 0x2a, 0xeb, 0x69, 0x5d, + 0x45, 0x39, 0x54, 0x04, 0xe7, 0x2e, 0x6c, 0xff, 0x92, 0x4a, 0xb5, 0x3b, 0x89, 0x43, 0x9e, 0xcc, + 0xd9, 0x22, 0x17, 0xbe, 0x11, 0x0a, 0xe7, 0xdf, 0x16, 0xec, 0x5c, 0x22, 0x50, 0x3c, 0x78, 0x0a, + 0xfd, 0xd8, 0xcf, 0x24, 0x15, 0x65, 0x95, 0x94, 0xc7, 0x55, 0x28, 0x5a, 0xd7, 0x41, 0xd1, 0x6e, + 0x40, 0x71, 0x0b, 0x7a, 0xb1, 0x7f, 0xe1, 0xc5, 0x27, 0xc5, 0x72, 0xd4, 0x8d, 0xfd, 0x8b, 0x97, + 0x27, 0xd8, 0xd9, 0x98, 0xf0, 0x4e, 0xf2, 0xe0, 0x8c, 0xca, 0xac, 0xea, 0x6c, 0x4c, 0x3c, 0xd1, + 0x14, 0xf5, 0x68, 0x25, 0xf0, 0x4d, 0x4e, 0x73, 0x9a, 0x15, 0xbd, 0x42, 0x0d, 0xc7, 0x5f, 0x21, + 0x01, 0x97, 0x29, 0x5c, 0x1d, 0xb1, 0x4b, 0x0c, 0xdc, 0xe2, 0x74, 0xf0, 0x97, 0x01, 0x8c, 0x8e, + 0xa9, 0x7f, 0x4e, 0x69, 0x88, 0x0f, 0x26, 0x8b, 0xb2, 0xd0, 0xea, 0xbf, 0x6c, 0xc9, 0xfd, 0xd5, + 0x8a, 0x5a, 0xfb, 0x53, 0x7a, 0xf6, 0xc9, 0x75, 0x62, 0x45, 0xce, 0xde, 0x20, 0xaf, 0x60, 0x68, + 0xfc, 0x74, 0x24, 0xdb, 0x86, 0x62, 0xe3, 0x17, 0xf1, 0x6c, 0xe7, 0x12, 0x6e, 0x69, 0xed, 0x91, + 0x45, 0x5e, 0xc0, 0xd0, 0x58, 0x69, 0x4c, 0x7b, 0xcd, 0xdd, 0xca, 0xb4, 0xb7, 0x66, 0x0f, 0x72, + 0x6e, 0x28, 0x6b, 0xc6, 0x62, 0x62, 0x5a, 0x6b, 0xae, 0x42, 0xa6, 0xb5, 0x75, 0xdb, 0x0c, 0x5a, + 0x33, 0xf6, 0x00, 0xd3, 0x5a, 0x73, 0xcb, 0x31, 0xad, 0xad, 0x59, 0x1e, 0x9c, 0x1b, 0xe4, 0x6b, + 0x98, 0x1c, 0x4b, 0x41, 0xfd, 0x78, 0xc9, 0x5e, 0x41, 0xf0, 0x7f, 0xb0, 0xba, 0x67, 0x3d, 0xb2, + 0xc8, 0xef, 0x60, 0xb3, 0x31, 0xe5, 0x89, 0xb3, 0xd4, 0xbc, 0x6c, 0x3d, 0x99, 0xdd, 0xbb, 0x52, + 0xa6, 0xf2, 0xfc, 0x2b, 0x18, 0x99, 0xc3, 0x95, 0x18, 0x4e, 0xad, 0xd9, 0x1f, 0x66, 0x77, 0x2f, + 0x63, 0x9b, 0x06, 0xcd, 0xfe, 0x6e, 0x1a, 0x5c, 0x33, 0xe1, 0x4c, 0x83, 0xeb, 0xc6, 0x82, 0x73, + 0x83, 0xfc, 0x16, 0x36, 0x56, 0xfb, 0x2c, 0xf9, 0x78, 0x15, 0xba, 0x46, 0xfb, 0x9e, 0x39, 0x57, + 0x89, 0x54, 0xc6, 0x9f, 0x03, 0x2c, 0xdb, 0x27, 0xb9, 0xb3, 0xd4, 0x69, 0xb4, 0xef, 0xd9, 0xf6, + 0x7a, 0x66, 0x65, 0xea, 0xf7, 0x70, 0x6b, 0x6d, 0x8f, 0x22, 0x46, 0x01, 0x5e, 0xd5, 0xe5, 0x66, + 0x3f, 0xba, 0x56, 0xae, 0xbc, 0xeb, 0xc9, 0x5d, 0xd8, 0xc8, 0x74, 0x8b, 0x98, 0x67, 0xfb, 0x41, + 0xc4, 0x68, 0x22, 0x9f, 0x00, 0x6a, 0xbc, 0x16, 0x5c, 0xf2, 0x93, 0x1e, 0xfe, 0xdd, 0xf6, 0xd3, + 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xf6, 0xb5, 0x1d, 0x19, 0x7d, 0x13, 0x00, 0x00, } diff --git a/weed/pb/master_pb/master.pb.go b/weed/pb/master_pb/master.pb.go index 95c9533a1..c33e2b768 100644 --- a/weed/pb/master_pb/master.pb.go +++ b/weed/pb/master_pb/master.pb.go @@ -428,12 +428,10 @@ type VolumeEcShardInformationMessage struct { EcIndexBits uint32 `protobuf:"varint,3,opt,name=ec_index_bits,json=ecIndexBits" json:"ec_index_bits,omitempty"` } -func (m *VolumeEcShardInformationMessage) Reset() { *m = VolumeEcShardInformationMessage{} } -func (m *VolumeEcShardInformationMessage) String() string { return proto.CompactTextString(m) } -func (*VolumeEcShardInformationMessage) ProtoMessage() {} -func (*VolumeEcShardInformationMessage) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{4} -} +func (m *VolumeEcShardInformationMessage) Reset() { *m = VolumeEcShardInformationMessage{} } +func (m *VolumeEcShardInformationMessage) String() string { return proto.CompactTextString(m) } +func (*VolumeEcShardInformationMessage) ProtoMessage() {} +func (*VolumeEcShardInformationMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } func (m *VolumeEcShardInformationMessage) GetId() uint32 { if m != nil { @@ -1424,12 +1422,10 @@ type GetMasterConfigurationResponse struct { MetricsIntervalSeconds uint32 `protobuf:"varint,2,opt,name=metrics_interval_seconds,json=metricsIntervalSeconds" json:"metrics_interval_seconds,omitempty"` } -func (m *GetMasterConfigurationResponse) Reset() { *m = GetMasterConfigurationResponse{} } -func (m *GetMasterConfigurationResponse) String() string { return proto.CompactTextString(m) } -func (*GetMasterConfigurationResponse) ProtoMessage() {} -func (*GetMasterConfigurationResponse) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{32} -} +func (m *GetMasterConfigurationResponse) Reset() { *m = GetMasterConfigurationResponse{} } +func (m *GetMasterConfigurationResponse) String() string { return proto.CompactTextString(m) } +func (*GetMasterConfigurationResponse) ProtoMessage() {} +func (*GetMasterConfigurationResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{32} } func (m *GetMasterConfigurationResponse) GetMetricsAddress() string { if m != nil { diff --git a/weed/pb/volume_server_pb/volume_server.pb.go b/weed/pb/volume_server_pb/volume_server.pb.go index 588b18f2e..56baa0cf7 100644 --- a/weed/pb/volume_server_pb/volume_server.pb.go +++ b/weed/pb/volume_server_pb/volume_server.pb.go @@ -1035,12 +1035,10 @@ func (m *VolumeEcShardsGenerateRequest) GetCollection() string { type VolumeEcShardsGenerateResponse struct { } -func (m *VolumeEcShardsGenerateResponse) Reset() { *m = VolumeEcShardsGenerateResponse{} } -func (m *VolumeEcShardsGenerateResponse) String() string { return proto.CompactTextString(m) } -func (*VolumeEcShardsGenerateResponse) ProtoMessage() {} -func (*VolumeEcShardsGenerateResponse) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{41} -} +func (m *VolumeEcShardsGenerateResponse) Reset() { *m = VolumeEcShardsGenerateResponse{} } +func (m *VolumeEcShardsGenerateResponse) String() string { return proto.CompactTextString(m) } +func (*VolumeEcShardsGenerateResponse) ProtoMessage() {} +func (*VolumeEcShardsGenerateResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{41} } type VolumeEcShardsRebuildRequest struct { VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` @@ -1413,12 +1411,10 @@ func (m *VolumeEcShardsToVolumeRequest) GetCollection() string { type VolumeEcShardsToVolumeResponse struct { } -func (m *VolumeEcShardsToVolumeResponse) Reset() { *m = VolumeEcShardsToVolumeResponse{} } -func (m *VolumeEcShardsToVolumeResponse) String() string { return proto.CompactTextString(m) } -func (*VolumeEcShardsToVolumeResponse) ProtoMessage() {} -func (*VolumeEcShardsToVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{57} -} +func (m *VolumeEcShardsToVolumeResponse) Reset() { *m = VolumeEcShardsToVolumeResponse{} } +func (m *VolumeEcShardsToVolumeResponse) String() string { return proto.CompactTextString(m) } +func (*VolumeEcShardsToVolumeResponse) ProtoMessage() {} +func (*VolumeEcShardsToVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{57} } type ReadVolumeFileStatusRequest struct { VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` @@ -2085,10 +2081,8 @@ type QueryRequest_InputSerialization_JSONInput struct { func (m *QueryRequest_InputSerialization_JSONInput) Reset() { *m = QueryRequest_InputSerialization_JSONInput{} } -func (m *QueryRequest_InputSerialization_JSONInput) String() string { - return proto.CompactTextString(m) -} -func (*QueryRequest_InputSerialization_JSONInput) ProtoMessage() {} +func (m *QueryRequest_InputSerialization_JSONInput) String() string { return proto.CompactTextString(m) } +func (*QueryRequest_InputSerialization_JSONInput) ProtoMessage() {} func (*QueryRequest_InputSerialization_JSONInput) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{70, 1, 1} } diff --git a/weed/replication/sink/azuresink/azure_sink.go b/weed/replication/sink/azuresink/azure_sink.go index f7c0eef0d..66e7b9c3e 100644 --- a/weed/replication/sink/azuresink/azure_sink.go +++ b/weed/replication/sink/azuresink/azure_sink.go @@ -115,7 +115,7 @@ func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry) error { } var writeErr error - _, readErr := util.ReadUrlAsStream(fileUrl, chunk.Offset, int(chunk.Size), func(data []byte) { + readErr := util.ReadUrlAsStream(fileUrl, nil, chunk.IsFullChunk, chunk.Offset, int(chunk.Size), func(data []byte) { _, writeErr = appendBlobURL.AppendBlock(context.Background(), bytes.NewReader(data), azblob.AppendBlobAccessConditions{}, nil) }) diff --git a/weed/replication/sink/b2sink/b2_sink.go b/weed/replication/sink/b2sink/b2_sink.go index e687170dd..0f8313638 100644 --- a/weed/replication/sink/b2sink/b2_sink.go +++ b/weed/replication/sink/b2sink/b2_sink.go @@ -103,7 +103,7 @@ func (g *B2Sink) CreateEntry(key string, entry *filer_pb.Entry) error { } var writeErr error - _, readErr := util.ReadUrlAsStream(fileUrl, chunk.Offset, int(chunk.Size), func(data []byte) { + readErr := util.ReadUrlAsStream(fileUrl, nil, chunk.IsFullChunk, chunk.Offset, int(chunk.Size), func(data []byte) { _, err := writer.Write(data) if err != nil { writeErr = err diff --git a/weed/replication/sink/filersink/fetch_write.go b/weed/replication/sink/filersink/fetch_write.go index 232b68fec..512fbd46b 100644 --- a/weed/replication/sink/filersink/fetch_write.go +++ b/weed/replication/sink/filersink/fetch_write.go @@ -50,6 +50,7 @@ func (fs *FilerSink) replicateOneChunk(sourceChunk *filer_pb.FileChunk, dir stri Mtime: sourceChunk.Mtime, ETag: sourceChunk.ETag, SourceFileId: sourceChunk.GetFileIdString(), + CipherKey: sourceChunk.CipherKey, }, nil } @@ -95,8 +96,8 @@ func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk, dir string) glog.V(4).Infof("replicating %s to %s header:%+v", filename, fileUrl, header) - uploadResult, err := operation.Upload(fileUrl, filename, readCloser, - "gzip" == header.Get("Content-Encoding"), header.Get("Content-Type"), nil, auth) + // fetch data as is, regardless whether it is encrypted or not + uploadResult, err := operation.Upload(fileUrl, filename, false, readCloser, "gzip" == header.Get("Content-Encoding"), header.Get("Content-Type"), nil, auth) if err != nil { glog.V(0).Infof("upload data %v to %s: %v", filename, fileUrl, err) return "", fmt.Errorf("upload data: %v", err) diff --git a/weed/replication/sink/gcssink/gcs_sink.go b/weed/replication/sink/gcssink/gcs_sink.go index d5b1e137f..dd8567e0e 100644 --- a/weed/replication/sink/gcssink/gcs_sink.go +++ b/weed/replication/sink/gcssink/gcs_sink.go @@ -6,13 +6,14 @@ import ( "os" "cloud.google.com/go/storage" + "google.golang.org/api/option" + "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/replication/sink" "github.com/chrislusf/seaweedfs/weed/replication/source" "github.com/chrislusf/seaweedfs/weed/util" - "google.golang.org/api/option" ) type GcsSink struct { @@ -100,7 +101,7 @@ func (g *GcsSink) CreateEntry(key string, entry *filer_pb.Entry) error { return err } - _, err = util.ReadUrlAsStream(fileUrl, chunk.Offset, int(chunk.Size), func(data []byte) { + err = util.ReadUrlAsStream(fileUrl, nil, chunk.IsFullChunk, chunk.Offset, int(chunk.Size), func(data []byte) { wc.Write(data) }) diff --git a/weed/replication/sink/s3sink/s3_write.go b/weed/replication/sink/s3sink/s3_write.go index 60885c30c..dcc041642 100644 --- a/weed/replication/sink/s3sink/s3_write.go +++ b/weed/replication/sink/s3sink/s3_write.go @@ -162,6 +162,6 @@ func (s3sink *S3Sink) buildReadSeeker(chunk *filer2.ChunkView) (io.ReadSeeker, e return nil, err } buf := make([]byte, chunk.Size) - util.ReadUrl(fileUrl, chunk.Offset, int(chunk.Size), buf, true) + util.ReadUrl(fileUrl, nil, false, chunk.Offset, int(chunk.Size), buf) return bytes.NewReader(buf), nil } diff --git a/weed/server/common.go b/weed/server/common.go index 31a9a73b8..d7ab8d1ee 100644 --- a/weed/server/common.go +++ b/weed/server/common.go @@ -134,7 +134,7 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl st } debug("upload file to store", url) - uploadResult, err := operation.Upload(url, fname, bytes.NewReader(data), isGzipped, mimeType, pairMap, assignResult.Auth) + uploadResult, err := operation.Upload(url, fname, false, bytes.NewReader(data), isGzipped, mimeType, pairMap, assignResult.Auth) if err != nil { writeJsonError(w, r, http.StatusInternalServerError, err) return diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 35539acca..b000bf3a6 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -338,5 +338,6 @@ func (fs *FilerServer) GetFilerConfiguration(ctx context.Context, req *filer_pb. MaxMb: uint32(fs.option.MaxMB), DirBuckets: fs.filer.DirBucketsPath, DirQueues: fs.filer.DirQueuesPath, + Cipher: fs.filer.Cipher, }, nil } diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 57caeb6d4..4c493b6b3 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -46,6 +46,7 @@ type FilerOption struct { DisableHttp bool Port uint32 recursiveDelete bool + Cipher bool } type FilerServer struct { @@ -67,6 +68,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) } fs.filer = filer2.NewFiler(option.Masters, fs.grpcDialOption, option.Port+10000) + fs.filer.Cipher = option.Cipher go fs.filer.KeepConnectedToMaster() diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index ba21298ba..aff2b9159 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -14,6 +14,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/util" ) @@ -93,7 +94,7 @@ func (fs *FilerServer) handleSingleChunk(w http.ResponseWriter, r *http.Request, return } - if fs.option.RedirectOnRead { + if fs.option.RedirectOnRead && entry.Chunks[0].CipherKey == nil { stats.FilerRequestCounter.WithLabelValues("redirect").Inc() http.Redirect(w, r, urlString, http.StatusFound) return @@ -136,7 +137,27 @@ func (fs *FilerServer) handleSingleChunk(w http.ResponseWriter, r *http.Request, w.Header().Set("Content-Type", entry.Attr.Mime) } w.WriteHeader(resp.StatusCode) - io.Copy(w, resp.Body) + if entry.Chunks[0].CipherKey == nil { + io.Copy(w, resp.Body) + } else { + fs.writeEncryptedChunk(w, resp, entry.Chunks[0]) + } +} + +func (fs *FilerServer) writeEncryptedChunk(w http.ResponseWriter, resp *http.Response, chunk *filer_pb.FileChunk) { + encryptedData, err := ioutil.ReadAll(resp.Body) + if err != nil { + glog.V(1).Infof("read encrypted %s failed, err: %v", chunk.FileId, err) + w.WriteHeader(http.StatusNotFound) + return + } + decryptedData, err := util.Decrypt(encryptedData, util.CipherKey(chunk.CipherKey)) + if err != nil { + glog.V(1).Infof("decrypt %s failed, err: %v", chunk.FileId, err) + w.WriteHeader(http.StatusNotFound) + return + } + w.Write(decryptedData) } func (fs *FilerServer) handleMultipleChunks(w http.ResponseWriter, r *http.Request, entry *filer2.Entry) { diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index 45cad480d..c8eadf82a 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -182,7 +182,7 @@ func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *ht stats.FilerRequestHistogram.WithLabelValues("postAutoChunkUpload").Observe(time.Since(start).Seconds()) }() - uploadResult, uploadError := operation.Upload(urlLocation, fileName, limitedReader, false, contentType, nil, auth) + uploadResult, uploadError := operation.Upload(urlLocation, fileName, fs.option.Cipher, limitedReader, false, contentType, nil, auth) if uploadError != nil { return 0, uploadError } diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go index a07f6be01..9451fdf99 100644 --- a/weed/server/webdav_server.go +++ b/weed/server/webdav_server.go @@ -32,6 +32,7 @@ type WebDavOption struct { Collection string Uid uint32 Gid uint32 + Cipher bool } type WebDavServer struct { @@ -418,7 +419,7 @@ func (f *WebDavFile) Write(buf []byte) (int, error) { fileUrl := fmt.Sprintf("http://%s/%s", host, fileId) bufReader := bytes.NewReader(buf) - uploadResult, err := operation.Upload(fileUrl, f.name, bufReader, false, "", nil, auth) + uploadResult, err := operation.Upload(fileUrl, f.name, f.fs.option.Cipher, bufReader, false, "", nil, auth) if err != nil { glog.V(0).Infof("upload data %v to %s: %v", f.name, fileUrl, err) return 0, fmt.Errorf("upload data: %v", err) @@ -429,11 +430,12 @@ func (f *WebDavFile) Write(buf []byte) (int, error) { } chunk := &filer_pb.FileChunk{ - FileId: fileId, - Offset: f.off, - Size: uint64(len(buf)), - Mtime: time.Now().UnixNano(), - ETag: uploadResult.ETag, + FileId: fileId, + Offset: f.off, + Size: uint64(len(buf)), + Mtime: time.Now().UnixNano(), + ETag: uploadResult.ETag, + CipherKey: uploadResult.CipherKey, } f.entry.Chunks = append(f.entry.Chunks, chunk) diff --git a/weed/topology/store_replicate.go b/weed/topology/store_replicate.go index b195b48ed..332ec4d65 100644 --- a/weed/topology/store_replicate.go +++ b/weed/topology/store_replicate.go @@ -72,9 +72,8 @@ func ReplicatedWrite(masterNode string, s *storage.Store, } } - _, err := operation.Upload(u.String(), - string(n.Name), bytes.NewReader(n.Data), n.IsGzipped(), string(n.Mime), - pairMap, jwt) + // volume server do not know about encryption + _, err := operation.Upload(u.String(), string(n.Name), false, bytes.NewReader(n.Data), n.IsGzipped(), string(n.Mime), pairMap, jwt) return err }); err != nil { size = 0 diff --git a/weed/util/cipher.go b/weed/util/cipher.go new file mode 100644 index 000000000..f044c2ca3 --- /dev/null +++ b/weed/util/cipher.go @@ -0,0 +1,60 @@ +package util + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "errors" + "io" + + "github.com/chrislusf/seaweedfs/weed/glog" +) + +type CipherKey []byte + +func GenCipherKey() CipherKey { + key := make([]byte, 32) + if _, err := io.ReadFull(rand.Reader, key); err != nil { + glog.Fatalf("random key gen: %v", err) + } + return CipherKey(key) +} + +func Encrypt(plaintext []byte, key CipherKey) ([]byte, error) { + c, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + + gcm, err := cipher.NewGCM(c) + if err != nil { + return nil, err + } + + nonce := make([]byte, gcm.NonceSize()) + if _, err = io.ReadFull(rand.Reader, nonce); err != nil { + return nil, err + } + + return gcm.Seal(nonce, nonce, plaintext, nil), nil +} + +func Decrypt(ciphertext []byte, key CipherKey) ([]byte, error) { + c, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + + gcm, err := cipher.NewGCM(c) + if err != nil { + return nil, err + } + + nonceSize := gcm.NonceSize() + if len(ciphertext) < nonceSize { + return nil, errors.New("ciphertext too short") + } + + nonce, ciphertext := ciphertext[:nonceSize], ciphertext[nonceSize:] + return gcm.Open(nil, nonce, ciphertext, nil) +} diff --git a/weed/util/http_util.go b/weed/util/http_util.go index f819d8497..833db910c 100644 --- a/weed/util/http_util.go +++ b/weed/util/http_util.go @@ -189,13 +189,21 @@ func NormalizeUrl(url string) string { return "http://" + url } -func ReadUrl(fileUrl string, offset int64, size int, buf []byte, isReadRange bool) (int64, error) { +func ReadUrl(fileUrl string, cipherKey []byte, isFullChunk bool, offset int64, size int, buf []byte) (int64, error) { + + if cipherKey != nil { + var n int + err := readEncryptedUrl(fileUrl, cipherKey, offset, size, func(data []byte) { + n = copy(buf, data) + }) + return int64(n), err + } req, err := http.NewRequest("GET", fileUrl, nil) if err != nil { return 0, err } - if isReadRange { + if !isFullChunk { req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size)-1)) } else { req.Header.Set("Accept-Encoding", "gzip") @@ -250,43 +258,64 @@ func ReadUrl(fileUrl string, offset int64, size int, buf []byte, isReadRange boo return n, err } -func ReadUrlAsStream(fileUrl string, offset int64, size int, fn func(data []byte)) (int64, error) { +func ReadUrlAsStream(fileUrl string, cipherKey []byte, isFullChunk bool, offset int64, size int, fn func(data []byte)) error { + + if cipherKey != nil { + return readEncryptedUrl(fileUrl, cipherKey, offset, size, fn) + } req, err := http.NewRequest("GET", fileUrl, nil) if err != nil { - return 0, err + return err + } + + if !isFullChunk { + req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size)-1)) } - req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size)-1)) r, err := client.Do(req) if err != nil { - return 0, err + return err } defer CloseResponse(r) if r.StatusCode >= 400 { - return 0, fmt.Errorf("%s: %s", fileUrl, r.Status) + return fmt.Errorf("%s: %s", fileUrl, r.Status) } var ( m int - n int64 ) buf := make([]byte, 64*1024) for { m, err = r.Body.Read(buf) fn(buf[:m]) - n += int64(m) if err == io.EOF { - return n, nil + return nil } if err != nil { - return n, err + return err } } } +func readEncryptedUrl(fileUrl string, cipherKey []byte, offset int64, size int, fn func(data []byte)) error { + encryptedData, err := Get(fileUrl) + if err != nil { + return fmt.Errorf("fetch %s: %v", fileUrl, err) + } + decryptedData, err := Decrypt(encryptedData, CipherKey(cipherKey)) + if err != nil { + return fmt.Errorf("decrypt %s: %v", fileUrl, err) + } + if len(decryptedData) < int(offset)+size { + return fmt.Errorf("read decrypted %s size %d [%d, %d)", fileUrl, len(decryptedData), offset, int(offset)+size) + } + fn(decryptedData[int(offset) : int(offset)+size]) + return nil +} + func ReadUrlAsReaderCloser(fileUrl string, rangeHeader string) (io.ReadCloser, error) { req, err := http.NewRequest("GET", fileUrl, nil) |