update ChunkedFile to seekable reader, so we can use io.* to read data

1 files changed, 148 insertions, 55 deletions

diff --git a/go/operation/chunked_file.go b/go/operation/chunked_file.go
index 953765149..ca97566a5 100644
--- a/go/operation/chunked_file.go
+++ b/go/operation/chunked_file.go
@@ -8,10 +8,18 @@ import (
 	"net/http"
 	"sort"
 
+	"sync"
+
+	"github.com/chrislusf/seaweedfs/go/glog"
 	"github.com/chrislusf/seaweedfs/go/util"
 )
 
-var ErrOutOfRange = errors.New("Out of Range")
+var (
+	// when the remote server does not allow range requests (Accept-Ranges was not set)
+	ErrRangeRequestsNotSupported = errors.New("Range requests are not supported by the remote server")
+	// ErrInvalidRange is returned by Read when trying to read past the end of the file
+	ErrInvalidRange = errors.New("Invalid range")
+)
 
 type ChunkInfo struct {
 	Fid    string `json:"fid,omitempty"`
@@ -21,41 +29,74 @@ type ChunkInfo struct {
 
 type ChunkList []*ChunkInfo
 
-type ChunkedFile struct {
-	Name   string    `json:"name,omitempty"`
-	Mime   string    `json:"mime,omitempty"`
-	Size   int64     `json:"size,omitempty"`
-	Chunks ChunkList `json:"chunks,omitempty"`
+type ChunkManifest struct {
+	Name      string    `json:"name,omitempty"`
+	Mime      string    `json:"mime,omitempty"`
+	Size      int64     `json:"size,omitempty"`
+	Chunks    ChunkList `json:"chunks,omitempty"`
+}
 
-	master string `json:"-"`
+// seekable chunked file reader
+type ChunkedFileReader struct {
+	Manifest *ChunkManifest
+	Master   string
+	pos      int64
+	pr       *io.PipeReader
+	pw       *io.PipeWriter
+	mutex    sync.Mutex
 }
 
 func (s ChunkList) Len() int           { return len(s) }
 func (s ChunkList) Less(i, j int) bool { return s[i].Offset < s[j].Offset }
 func (s ChunkList) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
 
-func NewChunkedNeedle(buffer []byte, master string) (*ChunkedFile, error) {
-	c := ChunkedFile{}
-
-	if e := json.Unmarshal(buffer, c); e != nil {
+func LoadChunkedManifest(buffer []byte) (*ChunkManifest, error) {
+	cm := ChunkManifest{}
+	if e := json.Unmarshal(buffer, cm); e != nil {
 		return nil, e
 	}
-	sort.Sort(c.Chunks)
-	c.master = master
-	return &c, nil
+	sort.Sort(cm.Chunks)
+	return &cm, nil
+}
+
+func (cm *ChunkManifest) GetData() ([]byte, error) {
+	return json.Marshal(cm)
+}
+
+func (cm *ChunkManifest) DeleteChunks(master string) error {
+	fileIds := make([]string, 0, len(cm.Chunks))
+	for _, ci := range cm.Chunks {
+		fileIds = append(fileIds, ci.Fid)
+	}
+	results, e := DeleteFiles(master, fileIds)
+	if e != nil {
+		return e
+	}
+	deleteError := 0
+	for _, ret := range results.Results {
+		if ret.Error != "" {
+			deleteError++
+			glog.V(0).Infoln("delete error:", ret.Error, ret.Fid)
+		}
+	}
+	if deleteError > 0 {
+		return errors.New("Not all chunks deleted.")
+	}
+	return nil
 }
 
-func (c *ChunkedFile) Marshal() ([]byte, error) {
-	return json.Marshal(c)
+func (cm *ChunkManifest) StoredHelper() error {
+	//TODO
+	return nil
 }
 
-func copyChunk(fileUrl string, w io.Writer, startOffset, size int64) (written int64, e error) {
+func httpRangeDownload(fileUrl string, w io.Writer, offset int64) (written int64, e error) {
 	req, err := http.NewRequest("GET", fileUrl, nil)
 	if err != nil {
 		return written, err
 	}
-	if startOffset > 0 {
-		req.Header.Set("Range", fmt.Sprintf("bytes=%d-", startOffset))
+	if offset > 0 {
+		req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset))
 	}
 
 	resp, err := util.Do(req)
@@ -63,65 +104,117 @@ func copyChunk(fileUrl string, w io.Writer, startOffset, size int64) (written in
 		return written, err
 	}
 	defer resp.Body.Close()
-	if startOffset > 0 && resp.StatusCode != 206 {
-		return written, fmt.Errorf("Cannot Read Needle Position: %d [%s]", startOffset, fileUrl)
-	}
 
-	if size > 0 {
-		return io.CopyN(w, resp.Body, size)
-	} else {
-		return io.Copy(w, resp.Body)
+	switch resp.StatusCode {
+	case http.StatusRequestedRangeNotSatisfiable:
+		return written, ErrInvalidRange
+	case http.StatusOK:
+		if offset > 0 {
+			return written, ErrRangeRequestsNotSupported
+		}
+	case http.StatusPartialContent:
+		break
+	default:
+		return written, fmt.Errorf("Read Needle http error: [%d] %s", resp.StatusCode, fileUrl)
+
 	}
+	return io.Copy(w, resp.Body)
 }
 
-func (c *ChunkedFile) WriteBuffer(w io.Writer, offset, size int64) (written int64, e error) {
-	if offset >= c.Size || offset+size > c.Size {
-		return written, ErrOutOfRange
+func (cf *ChunkedFileReader) Seek(offset int64, whence int) (int64, error) {
+	var err error
+	switch whence {
+	case 0:
+	case 1:
+		offset += cf.pos
+	case 2:
+		offset = cf.Manifest.Size - offset
+	}
+	if offset > cf.Manifest.Size {
+		err = ErrInvalidRange
 	}
+	if cf.pos != offset {
+		cf.Close()
+	}
+	cf.pos = offset
+	return cf.pos, err
+}
+
+func (cf *ChunkedFileReader) WriteTo(w io.Writer) (n int64, err error) {
+	cm := cf.Manifest
 	chunkIndex := -1
 	chunkStartOffset := int64(0)
-	for i, ci := range c.Chunks {
-		if offset >= ci.Offset && offset < ci.Offset+ci.Size {
+	for i, ci := range cm.Chunks {
+		if cf.pos >= ci.Offset && cf.pos < ci.Offset+ci.Size {
 			chunkIndex = i
-			chunkStartOffset = offset - ci.Offset
+			chunkStartOffset = cf.pos - ci.Offset
 			break
 		}
 	}
 	if chunkIndex < 0 {
-		return written, ErrOutOfRange
+		return n, ErrInvalidRange
 	}
-	//preload next chunk?
-	for ; chunkIndex < c.Chunks.Len(); chunkIndex++ {
-		ci := c.Chunks[chunkIndex]
-		fileUrl, lookupError := LookupFileId(c.master, ci.Fid)
+	for ; chunkIndex < cm.Chunks.Len(); chunkIndex++ {
+		ci := cm.Chunks[chunkIndex]
+		// if we need read date from local volume server first?
+		fileUrl, lookupError := LookupFileId(cf.Master, ci.Fid)
 		if lookupError != nil {
-			return written, lookupError
+			return n, lookupError
 		}
-		rsize := int64(0)
-		if size > 0 {
-			rsize = size - written
-		}
-		if n, e := copyChunk(fileUrl, w, chunkStartOffset, rsize); e != nil {
-			return written, e
+		if wn, e := httpRangeDownload(fileUrl, w, chunkStartOffset); e != nil {
+			return n, e
 		} else {
-			written += n
+			n += wn
+			cf.pos += wn
 		}
 
-		if size > 0 && written >= size {
-			break
-		}
 		chunkStartOffset = 0
 	}
+	return n, nil
+}
 
-	return written, nil
+func (cf *ChunkedFileReader) ReadAt(p []byte, off int64) (n int, err error) {
+	cf.Seek(off, 0)
+	return cf.Read(p)
 }
 
-func (c *ChunkedFile) DeleteHelper() error {
-	//TODO Delete all chunks
-	return nil
+func (cf *ChunkedFileReader) Read(p []byte) (int, error) {
+	return cf.getPipeReader().Read(p)
 }
 
-func (c *ChunkedFile) StoredHelper() error {
-	//TODO
-	return nil
+func (cf *ChunkedFileReader) Close() (e error) {
+	cf.mutex.Lock()
+	defer cf.mutex.Unlock()
+	return cf.closePipe()
+}
+
+func (cf *ChunkedFileReader) closePipe() (e error) {
+	if cf.pr != nil {
+		if err := cf.pr.Close(); err != nil {
+			e = err
+		}
+	}
+	cf.pr = nil
+	if cf.pw != nil {
+		if err := cf.pw.Close(); err != nil {
+			e = err
+		}
+	}
+	cf.pw = nil
+	return e
+}
+
+func (cf *ChunkedFileReader) getPipeReader() io.Reader {
+	cf.mutex.Lock()
+	defer cf.mutex.Unlock()
+	if cf.pr != nil && cf.pw != nil {
+		return cf.pr
+	}
+	cf.closePipe()
+	cf.pr, cf.pw = io.Pipe()
+	go func(pw *io.PipeWriter) {
+		_, e := cf.WriteTo(pw)
+		pw.CloseWithError(e)
+	}(cf.pw)
+	return cf.pr
 }