1 files changed, 84 insertions, 17 deletions

diff --git a/weed/util/compression.go b/weed/util/compression.go
index c6c9423e2..9d52810cb 100644
--- a/weed/util/compression.go
+++ b/weed/util/compression.go
@@ -4,54 +4,111 @@ import (
 	"bytes"
 	"compress/flate"
 	"compress/gzip"
+	"fmt"
 	"io/ioutil"
 	"strings"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"golang.org/x/tools/godoc/util"
+	// "github.com/klauspost/compress/zstd"
 )
 
+var (
+	UnsupportedCompression = fmt.Errorf("unsupported compression")
+)
+
+func MaybeGzipData(input []byte) []byte {
+	if IsGzippedContent(input) {
+		return input
+	}
+	gzipped, err := GzipData(input)
+	if err != nil {
+		return input
+	}
+	if len(gzipped)*10 > len(input)*9 {
+		return input
+	}
+	return gzipped
+}
+
+func MaybeDecompressData(input []byte) []byte {
+	uncompressed, err := DecompressData(input)
+	if err != nil {
+		if err != UnsupportedCompression {
+			glog.Errorf("decompressed data: %v", err)
+		}
+		return input
+	}
+	return uncompressed
+}
+
 func GzipData(input []byte) ([]byte, error) {
 	buf := new(bytes.Buffer)
 	w, _ := gzip.NewWriterLevel(buf, flate.BestSpeed)
 	if _, err := w.Write(input); err != nil {
-		glog.V(2).Infoln("error compressing data:", err)
+		glog.V(2).Infof("error gzip data: %v", err)
 		return nil, err
 	}
 	if err := w.Close(); err != nil {
-		glog.V(2).Infoln("error closing compressed data:", err)
+		glog.V(2).Infof("error closing gzipped data: %v", err)
 		return nil, err
 	}
 	return buf.Bytes(), nil
 }
-func UnGzipData(input []byte) ([]byte, error) {
+
+func DecompressData(input []byte) ([]byte, error) {
+	if IsGzippedContent(input) {
+		return ungzipData(input)
+	}
+	/*
+		if IsZstdContent(input) {
+			return unzstdData(input)
+		}
+	*/
+	return input, UnsupportedCompression
+}
+
+func ungzipData(input []byte) ([]byte, error) {
 	buf := bytes.NewBuffer(input)
 	r, _ := gzip.NewReader(buf)
 	defer r.Close()
 	output, err := ioutil.ReadAll(r)
 	if err != nil {
-		glog.V(2).Infoln("error uncompressing data:", err)
+		glog.V(2).Infof("error ungzip data: %v", err)
 	}
 	return output, err
 }
 
+func IsGzippedContent(data []byte) bool {
+	if len(data) < 2 {
+		return false
+	}
+	return data[0] == 31 && data[1] == 139
+}
+
 /*
-* Default more not to gzip since gzip can be done on client side.
- */func IsGzippable(ext, mtype string, data []byte) bool {
+var zstdEncoder, _ = zstd.NewWriter(nil)
 
-	shouldBeZipped, iAmSure := IsGzippableFileType(ext, mtype)
-	if iAmSure {
-		return shouldBeZipped
-	}
+func ZstdData(input []byte) ([]byte, error) {
+	return zstdEncoder.EncodeAll(input, nil), nil
+}
 
-	isMostlyText := util.IsText(data)
+var decoder, _ = zstd.NewReader(nil)
 
-	return isMostlyText
+func unzstdData(input []byte) ([]byte, error) {
+	return decoder.DecodeAll(input, nil)
 }
 
+func IsZstdContent(data []byte) bool {
+	if len(data) < 4 {
+		return false
+	}
+	return data[3] == 0xFD && data[2] == 0x2F && data[1] == 0xB5 && data[0] == 0x28
+}
+*/
+
 /*
-* Default more not to gzip since gzip can be done on client side.
- */func IsGzippableFileType(ext, mtype string) (shouldBeZipped, iAmSure bool) {
+* Default not to compressed since compression can be done on client side.
+ */func IsCompressableFileType(ext, mtype string) (shouldBeCompressed, iAmSure bool) {
 
 	// text
 	if strings.HasPrefix(mtype, "text/") {
@@ -60,7 +117,7 @@ func UnGzipData(input []byte) ([]byte, error) {
 
 	// images
 	switch ext {
-	case ".svg", ".bmp":
+	case ".svg", ".bmp", ".wav":
 		return true, true
 	}
 	if strings.HasPrefix(mtype, "image/") {
@@ -69,7 +126,7 @@ func UnGzipData(input []byte) ([]byte, error) {
 
 	// by file name extension
 	switch ext {
-	case ".zip", ".rar", ".gz", ".bz2", ".xz":
+	case ".zip", ".rar", ".gz", ".bz2", ".xz", ".zst":
 		return false, true
 	case ".pdf", ".txt", ".html", ".htm", ".css", ".js", ".json":
 		return true, true
@@ -81,6 +138,9 @@ func UnGzipData(input []byte) ([]byte, error) {
 
 	// by mime type
 	if strings.HasPrefix(mtype, "application/") {
+		if strings.HasSuffix(mtype, "zstd") {
+			return false, true
+		}
 		if strings.HasSuffix(mtype, "xml") {
 			return true, true
 		}
@@ -89,5 +149,12 @@ func UnGzipData(input []byte) ([]byte, error) {
 		}
 	}
 
+	if strings.HasPrefix(mtype, "audio/") {
+		switch strings.TrimPrefix(mtype, "audio/") {
+		case "wave", "wav", "x-wav", "x-pn-wav":
+			return true, true
+		}
+	}
+
 	return false, false
 }