Merge pull request #10 from chrislusf/master

sync

7 files changed, 70 insertions, 29 deletions

diff --git a/weed/operation/chunked_file.go b/weed/operation/chunked_file.go
index 653b7bf13..1bac028ff 100644
--- a/weed/operation/chunked_file.go
+++ b/weed/operation/chunked_file.go
@@ -57,7 +57,7 @@ func LoadChunkManifest(buffer []byte, isCompressed bool) (*ChunkManifest, error)
 	if isCompressed {
 		var err error
 		if buffer, err = util.DecompressData(buffer); err != nil {
-			return nil, err
+			glog.V(0).Infof("fail to decompress chunk manifest: %v", err)
 		}
 	}
 	cm := ChunkManifest{}
diff --git a/weed/server/volume_server_handlers_read.go b/weed/server/volume_server_handlers_read.go
index 89b7445e9..d730600e4 100644
--- a/weed/server/volume_server_handlers_read.go
+++ b/weed/server/volume_server_handlers_read.go
@@ -26,6 +26,8 @@ var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
 
 func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) {
 
+	// println(r.Method + " " + r.URL.Path)
+
 	stats.VolumeServerRequestCounter.WithLabelValues("get").Inc()
 	start := time.Now()
 	defer func() { stats.VolumeServerRequestHistogram.WithLabelValues("get").Observe(time.Since(start).Seconds()) }()
@@ -142,20 +144,18 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
 		}
 	}
 
-	if ext != ".gz" && ext != ".zst" {
-		if n.IsCompressed() {
-			if _, _, _, shouldResize := shouldResizeImages(ext, r); shouldResize {
-				if n.Data, err = util.DecompressData(n.Data); err != nil {
-					glog.V(0).Infoln("ungzip error:", err, r.URL.Path)
-				}
-			} else if strings.Contains(r.Header.Get("Accept-Encoding"), "zstd") && util.IsZstdContent(n.Data) {
-				w.Header().Set("Content-Encoding", "zstd")
-			} else if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") && util.IsGzippedContent(n.Data) {
-				w.Header().Set("Content-Encoding", "gzip")
-			} else {
-				if n.Data, err = util.DecompressData(n.Data); err != nil {
-					glog.V(0).Infoln("uncompress error:", err, r.URL.Path)
-				}
+	if n.IsCompressed() {
+		if _, _, _, shouldResize := shouldResizeImages(ext, r); shouldResize {
+			if n.Data, err = util.DecompressData(n.Data); err != nil {
+				glog.V(0).Infoln("ungzip error:", err, r.URL.Path)
+			}
+		} else if strings.Contains(r.Header.Get("Accept-Encoding"), "zstd") && util.IsZstdContent(n.Data) {
+			w.Header().Set("Content-Encoding", "zstd")
+		} else if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") && util.IsGzippedContent(n.Data) {
+			w.Header().Set("Content-Encoding", "gzip")
+		} else {
+			if n.Data, err = util.DecompressData(n.Data); err != nil {
+				glog.V(0).Infoln("uncompress error:", err, r.URL.Path)
 			}
 		}
 	}
diff --git a/weed/stats/disk.go b/weed/stats/disk.go
index 8af7240a8..a8f906213 100644
--- a/weed/stats/disk.go
+++ b/weed/stats/disk.go
@@ -8,6 +8,8 @@ import (
 func NewDiskStatus(path string) (disk *volume_server_pb.DiskStatus) {
 	disk = &volume_server_pb.DiskStatus{Dir: path}
 	fillInDiskStatus(disk)
-	glog.V(2).Infof("read disk size: %v", disk)
+	if disk.PercentUsed > 95 {
+		glog.V(0).Infof("disk status: %v", disk)
+	}
 	return
 }
diff --git a/weed/storage/needle/needle.go b/weed/storage/needle/needle.go
index 7d02758d6..150d6ee4b 100644
--- a/weed/storage/needle/needle.go
+++ b/weed/storage/needle/needle.go
@@ -44,7 +44,7 @@ type Needle struct {
 }
 
 func (n *Needle) String() (str string) {
-	str = fmt.Sprintf("%s Size:%d, DataSize:%d, Name:%s, Mime:%s", formatNeedleIdCookie(n.Id, n.Cookie), n.Size, n.DataSize, n.Name, n.Mime)
+	str = fmt.Sprintf("%s Size:%d, DataSize:%d, Name:%s, Mime:%s Compressed:%v", formatNeedleIdCookie(n.Id, n.Cookie), n.Size, n.DataSize, n.Name, n.Mime, n.IsCompressed())
 	return
 }
 
@@ -81,6 +81,7 @@ func CreateNeedleFromRequest(r *http.Request, fixJpgOrientation bool, sizeLimit
 		}
 	}
 	if pu.IsGzipped {
+		// println(r.URL.Path, "is set to compressed", pu.FileName, pu.IsGzipped, "dataSize", pu.OriginalDataSize)
 		n.SetIsCompressed()
 	}
 	if n.LastModified == 0 {
diff --git a/weed/storage/needle/needle_parse_upload.go b/weed/storage/needle/needle_parse_upload.go
index 785217761..dd678f87f 100644
--- a/weed/storage/needle/needle_parse_upload.go
+++ b/weed/storage/needle/needle_parse_upload.go
@@ -54,7 +54,7 @@ func ParseUpload(r *http.Request, sizeLimit int64) (pu *ParsedUpload, e error) {
 
 	pu.OriginalDataSize = len(pu.Data)
 	pu.UncompressedData = pu.Data
-	// println("received data", len(pu.Data), "isGzipped", pu.IsCompressed, "mime", pu.MimeType, "name", pu.FileName)
+	// println("received data", len(pu.Data), "isGzipped", pu.IsGzipped, "mime", pu.MimeType, "name", pu.FileName)
 	if pu.IsGzipped {
 		if unzipped, e := util.DecompressData(pu.Data); e == nil {
 			pu.OriginalDataSize = len(unzipped)
@@ -72,7 +72,7 @@ func ParseUpload(r *http.Request, sizeLimit int64) (pu *ParsedUpload, e error) {
 			mimeType = ""
 		}
 		if shouldBeCompressed, iAmSure := util.IsCompressableFileType(ext, mimeType); mimeType == "" && !iAmSure || shouldBeCompressed && iAmSure {
-			// println("ext", ext, "iAmSure", iAmSure, "shouldGzip", shouldGzip, "mimeType", pu.MimeType)
+			// println("ext", ext, "iAmSure", iAmSure, "shouldBeCompressed", shouldBeCompressed, "mimeType", pu.MimeType)
 			if compressedData, err := util.GzipData(pu.Data); err == nil {
 				if len(compressedData)*10 < len(pu.Data)*9 {
 					pu.Data = compressedData
diff --git a/weed/util/compression.go b/weed/util/compression.go
index 4488e019e..2881a7bfd 100644
--- a/weed/util/compression.go
+++ b/weed/util/compression.go
@@ -39,7 +39,7 @@ func DecompressData(input []byte) ([]byte, error) {
 	if IsZstdContent(input) {
 		return unzstdData(input)
 	}
-	return nil, fmt.Errorf("unsupported compression")
+	return input, fmt.Errorf("unsupported compression")
 }
 
 func ungzipData(input []byte) ([]byte, error) {
diff --git a/weed/util/http_util.go b/weed/util/http_util.go
index c67eb3276..5159fcd17 100644
--- a/weed/util/http_util.go
+++ b/weed/util/http_util.go
@@ -68,14 +68,28 @@ func Post(url string, values url.Values) ([]byte, error) {
 //	github.com/chrislusf/seaweedfs/unmaintained/repeated_vacuum/repeated_vacuum.go
 //	may need increasing http.Client.Timeout
 func Get(url string) ([]byte, error) {
-	r, err := client.Get(url)
+
+	request, err := http.NewRequest("GET", url, nil)
+	request.Header.Add("Accept-Encoding", "gzip")
+
+	response, err := client.Do(request)
 	if err != nil {
 		return nil, err
 	}
-	defer r.Body.Close()
-	b, err := ioutil.ReadAll(r.Body)
-	if r.StatusCode >= 400 {
-		return nil, fmt.Errorf("%s: %s", url, r.Status)
+	defer response.Body.Close()
+
+	var reader io.ReadCloser
+	switch response.Header.Get("Content-Encoding") {
+	case "gzip":
+		reader, err = gzip.NewReader(response.Body)
+		defer reader.Close()
+	default:
+		reader = response.Body
+	}
+
+	b, err := ioutil.ReadAll(reader)
+	if response.StatusCode >= 400 {
+		return nil, fmt.Errorf("%s: %s", url, response.Status)
 	}
 	if err != nil {
 		return nil, err
@@ -269,7 +283,9 @@ func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, is
 		return err
 	}
 
-	if !isFullChunk {
+	if isFullChunk {
+		req.Header.Add("Accept-Encoding", "gzip")
+	} else {
 		req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size)-1))
 	}
 
@@ -282,13 +298,23 @@ func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, is
 		return fmt.Errorf("%s: %s", fileUrl, r.Status)
 	}
 
+	var reader io.ReadCloser
+	contentEncoding := r.Header.Get("Content-Encoding")
+	switch contentEncoding {
+	case "gzip":
+		reader, err = gzip.NewReader(r.Body)
+		defer reader.Close()
+	default:
+		reader = r.Body
+	}
+
 	var (
 		m int
 	)
 	buf := make([]byte, 64*1024)
 
 	for {
-		m, err = r.Body.Read(buf)
+		m, err = reader.Read(buf)
 		fn(buf[:m])
 		if err == io.EOF {
 			return nil
@@ -312,7 +338,7 @@ func readEncryptedUrl(fileUrl string, cipherKey []byte, isContentCompressed bool
 	if isContentCompressed {
 		decryptedData, err = DecompressData(decryptedData)
 		if err != nil {
-			return fmt.Errorf("unzip decrypt %s: %v", fileUrl, err)
+			glog.V(0).Infof("unzip decrypt %s: %v", fileUrl, err)
 		}
 	}
 	if len(decryptedData) < int(offset)+size {
@@ -334,6 +360,8 @@ func ReadUrlAsReaderCloser(fileUrl string, rangeHeader string) (io.ReadCloser, e
 	}
 	if rangeHeader != "" {
 		req.Header.Add("Range", rangeHeader)
+	} else {
+		req.Header.Add("Accept-Encoding", "gzip")
 	}
 
 	r, err := client.Do(req)
@@ -344,7 +372,17 @@ func ReadUrlAsReaderCloser(fileUrl string, rangeHeader string) (io.ReadCloser, e
 		return nil, fmt.Errorf("%s: %s", fileUrl, r.Status)
 	}
 
-	return r.Body, nil
+	var reader io.ReadCloser
+	contentEncoding := r.Header.Get("Content-Encoding")
+	switch contentEncoding {
+	case "gzip":
+		reader, err = gzip.NewReader(r.Body)
+		defer reader.Close()
+	default:
+		reader = r.Body
+	}
+
+	return reader, nil
 }
 
 func CloseResponse(resp *http.Response) {