Merge pull request #224 from tnextday/feature/chunked-file-support

Feature/chunked file support

3 files changed, 20 insertions, 64 deletions

diff --git a/go/storage/compress.go b/go/storage/compress.go
deleted file mode 100644
index 4047c1723..000000000
--- a/go/storage/compress.go
+++ /dev/null
@@ -1,59 +0,0 @@
-package storage
-
-import (
-	"bytes"
-	"compress/flate"
-	"compress/gzip"
-	"io/ioutil"
-	"strings"
-
-	"github.com/chrislusf/seaweedfs/go/glog"
-)
-
-/*
-* Default more not to gzip since gzip can be done on client side.
- */
-func IsGzippable(ext, mtype string) bool {
-	if strings.HasPrefix(mtype, "text/") {
-		return true
-	}
-	switch ext {
-	case ".zip", ".rar", ".gz", ".bz2", ".xz":
-		return false
-	case ".pdf", ".txt", ".html", ".htm", ".css", ".js", ".json":
-		return true
-	}
-	if strings.HasPrefix(mtype, "application/") {
-		if strings.HasSuffix(mtype, "xml") {
-			return true
-		}
-		if strings.HasSuffix(mtype, "script") {
-			return true
-		}
-	}
-	return false
-}
-
-func GzipData(input []byte) ([]byte, error) {
-	buf := new(bytes.Buffer)
-	w, _ := gzip.NewWriterLevel(buf, flate.BestCompression)
-	if _, err := w.Write(input); err != nil {
-		glog.V(2).Infoln("error compressing data:", err)
-		return nil, err
-	}
-	if err := w.Close(); err != nil {
-		glog.V(2).Infoln("error closing compressed data:", err)
-		return nil, err
-	}
-	return buf.Bytes(), nil
-}
-func UnGzipData(input []byte) ([]byte, error) {
-	buf := bytes.NewBuffer(input)
-	r, _ := gzip.NewReader(buf)
-	defer r.Close()
-	output, err := ioutil.ReadAll(r)
-	if err != nil {
-		glog.V(2).Infoln("error uncompressing data:", err)
-	}
-	return output, err
-}
diff --git a/go/storage/needle.go b/go/storage/needle.go
index 04a9dc78d..32ebdae7d 100644
--- a/go/storage/needle.go
+++ b/go/storage/needle.go
@@ -15,6 +15,7 @@ import (
 	"github.com/chrislusf/seaweedfs/go/glog"
 	"github.com/chrislusf/seaweedfs/go/images"
 	"github.com/chrislusf/seaweedfs/go/util"
+	"github.com/chrislusf/seaweedfs/go/operation"
 )
 
 const (
@@ -52,7 +53,7 @@ func (n *Needle) String() (str string) {
 	return
 }
 
-func ParseUpload(r *http.Request) (fileName string, data []byte, mimeType string, isGzipped bool, modifiedTime uint64, ttl *TTL, e error) {
+func ParseUpload(r *http.Request) (fileName string, data []byte, mimeType string, isGzipped bool, modifiedTime uint64, ttl *TTL, isChunkedFile bool, e error) {
 	form, fe := r.MultipartReader()
 	if fe != nil {
 		glog.V(0).Infoln("MultipartReader [ERROR]", fe)
@@ -117,8 +118,8 @@ func ParseUpload(r *http.Request) (fileName string, data []byte, mimeType string
 	}
 	if part.Header.Get("Content-Encoding") == "gzip" {
 		isGzipped = true
-	} else if IsGzippable(ext, mtype) {
-		if data, e = GzipData(data); e != nil {
+	} else if operation.IsGzippable(ext, mtype) {
+		if data, e = operation.GzipData(data); e != nil {
 			return
 		}
 		isGzipped = true
@@ -132,12 +133,13 @@ func ParseUpload(r *http.Request) (fileName string, data []byte, mimeType string
 	}
 	modifiedTime, _ = strconv.ParseUint(r.FormValue("ts"), 10, 64)
 	ttl, _ = ReadTTL(r.FormValue("ttl"))
+	isChunkedFile, _ = strconv.ParseBool(r.FormValue("cm"))
 	return
 }
 func NewNeedle(r *http.Request, fixJpgOrientation bool) (n *Needle, e error) {
-	fname, mimeType, isGzipped := "", "", false
+	fname, mimeType, isGzipped, isChunkedFile := "", "", false, false
 	n = new(Needle)
-	fname, n.Data, mimeType, isGzipped, n.LastModified, n.Ttl, e = ParseUpload(r)
+	fname, n.Data, mimeType, isGzipped, n.LastModified, n.Ttl, isChunkedFile, e = ParseUpload(r)
 	if e != nil {
 		return
 	}
@@ -160,6 +162,10 @@ func NewNeedle(r *http.Request, fixJpgOrientation bool) (n *Needle, e error) {
 		n.SetHasTtl()
 	}
 
+	if isChunkedFile {
+		n.SetChunkManifest()
+	}
+
 	if fixJpgOrientation {
 		loweredName := strings.ToLower(fname)
 		if mimeType == "image/jpeg" || strings.HasSuffix(loweredName, ".jpg") || strings.HasSuffix(loweredName, ".jpeg") {
diff --git a/go/storage/needle_read_write.go b/go/storage/needle_read_write.go
index eb2d8d459..9d7af600a 100644
--- a/go/storage/needle_read_write.go
+++ b/go/storage/needle_read_write.go
@@ -16,6 +16,7 @@ const (
 	FlagHasMime             = 0x04
 	FlagHasLastModifiedDate = 0x08
 	FlagHasTtl              = 0x10
+	FlagChunkManifest       = 0x80
 	LastModifiedBytesLength = 5
 	TtlBytesLength          = 2
 )
@@ -280,3 +281,11 @@ func (n *Needle) HasTtl() bool {
 func (n *Needle) SetHasTtl() {
 	n.Flags = n.Flags | FlagHasTtl
 }
+
+func (n *Needle) IsChunkedManifest() bool {
+	return n.Flags&FlagChunkManifest > 0
+}
+
+func (n *Needle) SetChunkManifest() {
+	n.Flags = n.Flags | FlagChunkManifest
+}