prepare for zstd

1 files changed, 20 insertions, 18 deletions

diff --git a/weed/util/compression.go b/weed/util/compression.go
index 003f06cbf..de6bf0800 100644
--- a/weed/util/compression.go
+++ b/weed/util/compression.go
@@ -8,9 +8,8 @@ import (
 	"io/ioutil"
 	"strings"
 
-	"golang.org/x/tools/godoc/util"
-
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/klauspost/compress/zstd"
 )
 
 func GzipData(input []byte) ([]byte, error) {
@@ -30,6 +29,9 @@ func DecompressData(input []byte) ([]byte, error) {
 	if IsGzippedContent(input) {
 		return ungzipData(input)
 	}
+	if IsZstdContent(input) {
+		return unzstdData(input)
+	}
 	return nil, fmt.Errorf("unsupported compression")
 }
 
@@ -44,19 +46,10 @@ func ungzipData(input []byte) ([]byte, error) {
 	return output, err
 }
 
-/*
-* Default more not to gzip since gzip can be done on client side.
- */
-func IsGzippable(ext, mtype string, data []byte) bool {
-
-	shouldBeZipped, iAmSure := IsGzippableFileType(ext, mtype)
-	if iAmSure {
-		return shouldBeZipped
-	}
-
-	isMostlyText := util.IsText(data)
+var zstdEncoder, _ = zstd.NewWriter(nil)
 
-	return isMostlyText
+func unzstdData(input []byte) ([]byte, error) {
+	return zstdEncoder.EncodeAll(input, nil), nil
 }
 
 func IsGzippedContent(data []byte) bool {
@@ -66,9 +59,16 @@ func IsGzippedContent(data []byte) bool {
 	return data[0] == 31 && data[1] == 139
 }
 
+func IsZstdContent(data []byte) bool {
+	if len(data) < 4 {
+		return false
+	}
+	return data[0] == 0xFD && data[1] == 0x2F && data[2] == 0xB5 && data[3] == 0x28
+}
+
 /*
-* Default more not to gzip since gzip can be done on client side.
- */func IsGzippableFileType(ext, mtype string) (shouldBeZipped, iAmSure bool) {
+* Default not to compressed since compression can be done on client side.
+ */func IsCompressableFileType(ext, mtype string) (shouldBeCompressed, iAmSure bool) {
 
 	// text
 	if strings.HasPrefix(mtype, "text/") {
@@ -86,7 +86,7 @@ func IsGzippedContent(data []byte) bool {
 
 	// by file name extension
 	switch ext {
-	case ".zip", ".rar", ".gz", ".bz2", ".xz":
+	case ".zip", ".rar", ".gz", ".bz2", ".xz", ".zst":
 		return false, true
 	case ".pdf", ".txt", ".html", ".htm", ".css", ".js", ".json":
 		return true, true
@@ -98,13 +98,15 @@ func IsGzippedContent(data []byte) bool {
 
 	// by mime type
 	if strings.HasPrefix(mtype, "application/") {
+		if strings.HasSuffix(mtype, "zstd") {
+			return false, true
+		}
 		if strings.HasSuffix(mtype, "xml") {
 			return true, true
 		}
 		if strings.HasSuffix(mtype, "script") {
 			return true, true
 		}
-
 	}
 
 	if strings.HasPrefix(mtype, "audio/") {