diff options
| -rw-r--r-- | weed/command/filer_cat.go | 2 | ||||
| -rw-r--r-- | weed/filer/filer_on_meta_event.go | 2 | ||||
| -rw-r--r-- | weed/filer/read_write.go | 2 | ||||
| -rw-r--r-- | weed/filer/stream.go | 22 | ||||
| -rw-r--r-- | weed/server/filer_server_handlers_read.go | 10 | ||||
| -rw-r--r-- | weed/shell/command_fs_cat.go | 2 |
6 files changed, 17 insertions, 23 deletions
diff --git a/weed/command/filer_cat.go b/weed/command/filer_cat.go index c4281feba..a46098b04 100644 --- a/weed/command/filer_cat.go +++ b/weed/command/filer_cat.go @@ -110,7 +110,7 @@ func runFilerCat(cmd *Command, args []string) bool { filerCat.filerClient = client - return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64, false) + return filer.StreamContent(&filerCat, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64) }) diff --git a/weed/filer/filer_on_meta_event.go b/weed/filer/filer_on_meta_event.go index a91faeb24..c9f75a5ca 100644 --- a/weed/filer/filer_on_meta_event.go +++ b/weed/filer/filer_on_meta_event.go @@ -52,7 +52,7 @@ func (f *Filer) maybeReloadFilerConfiguration(event *filer_pb.SubscribeMetadataR func (f *Filer) readEntry(chunks []*filer_pb.FileChunk) ([]byte, error) { var buf bytes.Buffer - err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64, false) + err := StreamContent(f.MasterClient, &buf, chunks, 0, math.MaxInt64) if err != nil { return nil, err } diff --git a/weed/filer/read_write.go b/weed/filer/read_write.go index d92d526d5..c4c90fb63 100644 --- a/weed/filer/read_write.go +++ b/weed/filer/read_write.go @@ -27,7 +27,7 @@ func ReadEntry(masterClient *wdclient.MasterClient, filerClient filer_pb.Seaweed return err } - return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64, false) + return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64) } diff --git a/weed/filer/stream.go b/weed/filer/stream.go index f1181740e..3fac0895e 100644 --- a/weed/filer/stream.go +++ b/weed/filer/stream.go @@ -7,14 +7,16 @@ import ( "io" "math" "strings" + "time" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/wdclient" ) -func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64, isCheck bool) error { +func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error { glog.V(9).Infof("start to stream content for chunks: %+v\n", chunks) chunkViews := ViewFromChunks(masterClient.GetLookupFileIdFunction(), chunks, offset, size) @@ -34,31 +36,23 @@ func StreamContent(masterClient wdclient.HasLookupFileIdFunction, w io.Writer, c fileId2Url[chunkView.FileId] = urlStrings } - if isCheck { - // Pre-check all chunkViews urls - gErr := new(errgroup.Group) - CheckAllChunkViews(chunkViews, &fileId2Url, gErr) - if err := gErr.Wait(); err != nil { - glog.Errorf("check all chunks: %v", err) - return fmt.Errorf("check all chunks: %v", err) - } - return nil - } - for _, chunkView := range chunkViews { urlStrings := fileId2Url[chunkView.FileId] + start := time.Now() data, err := retriedFetchChunkData(urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size)) + stats.FilerRequestHistogram.WithLabelValues("chunkDownload").Observe(time.Since(start).Seconds()) if err != nil { - glog.Errorf("read chunk: %v", err) + stats.FilerRequestCounter.WithLabelValues("chunkDownloadError").Inc() return fmt.Errorf("read chunk: %v", err) } _, err = w.Write(data) if err != nil { - glog.Errorf("write chunk: %v", err) + stats.FilerRequestCounter.WithLabelValues("chunkDownloadedError").Inc() return fmt.Errorf("write chunk: %v", err) } + stats.FilerRequestCounter.WithLabelValues("chunkDownload").Inc() } return nil diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 6bc09e953..286523098 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -131,9 +131,6 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, if r.Method == "HEAD" { w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10)) - processRangeRequest(r, w, totalSize, mimeType, func(writer io.Writer, offset int64, size int64) error { - return filer.StreamContent(fs.filer.MasterClient, writer, entry.Chunks, offset, size, true) - }) return } @@ -161,7 +158,10 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, } return err } - return filer.StreamContent(fs.filer.MasterClient, writer, entry.Chunks, offset, size, false) + err = filer.StreamContent(fs.filer.MasterClient, writer, entry.Chunks, offset, size) + if err != nil { + glog.Errorf("failed to stream content %s: %v", r.URL, err) + } + return err }) - } diff --git a/weed/shell/command_fs_cat.go b/weed/shell/command_fs_cat.go index df43d93dc..3c5e13663 100644 --- a/weed/shell/command_fs_cat.go +++ b/weed/shell/command_fs_cat.go @@ -52,7 +52,7 @@ func (c *commandFsCat) Do(args []string, commandEnv *CommandEnv, writer io.Write return err } - return filer.StreamContent(commandEnv.MasterClient, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64, false) + return filer.StreamContent(commandEnv.MasterClient, writer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64) }) |