aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--weed/operation/upload_content.go8
-rw-r--r--weed/server/volume_server_handlers_read.go20
-rw-r--r--weed/storage/needle/needle_parse_upload.go5
-rw-r--r--weed/util/compression.go38
4 files changed, 37 insertions, 34 deletions
diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go
index 1ed240836..05b016457 100644
--- a/weed/operation/upload_content.go
+++ b/weed/operation/upload_content.go
@@ -98,7 +98,7 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i
mtype = ""
}
}
- if shouldBeZipped, iAmSure := util.IsGzippableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeZipped {
+ if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeCompressed {
shouldGzipNow = true
} else if !iAmSure && mtype == "" && len(data) > 128 {
var compressed []byte
@@ -142,7 +142,7 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i
uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
_, err = w.Write(encryptedData)
return
- }, "", false, "", nil, jwt)
+ }, "", false, len(encryptedData), "", nil, jwt)
if uploadResult != nil {
uploadResult.Name = filename
uploadResult.Mime = mtype
@@ -153,7 +153,7 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i
uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
_, err = w.Write(data)
return
- }, filename, contentIsGzipped, mtype, pairMap, jwt)
+ }, filename, contentIsGzipped, 0, mtype, pairMap, jwt)
}
if uploadResult == nil {
@@ -168,7 +168,7 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i
return uploadResult, err
}
-func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
+func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, originalDataSize int, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
body_buf := bytes.NewBufferString("")
body_writer := multipart.NewWriter(body_buf)
h := make(textproto.MIMEHeader)
diff --git a/weed/server/volume_server_handlers_read.go b/weed/server/volume_server_handlers_read.go
index aff1eab8d..7ef1170b3 100644
--- a/weed/server/volume_server_handlers_read.go
+++ b/weed/server/volume_server_handlers_read.go
@@ -142,21 +142,19 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
}
}
- if ext != ".gz" {
+ if ext != ".gz" && ext != ".zst" {
if n.IsCompressed() {
- if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
- if _, _, _, shouldResize := shouldResizeImages(ext, r); shouldResize {
- if n.Data, err = util.DecompressData(n.Data); err != nil {
- glog.V(0).Infoln("ungzip error:", err, r.URL.Path)
- }
- } else {
- if util.IsGzippedContent(n.Data) {
- w.Header().Set("Content-Encoding", "gzip")
- }
+ if _, _, _, shouldResize := shouldResizeImages(ext, r); shouldResize {
+ if n.Data, err = util.DecompressData(n.Data); err != nil {
+ glog.V(0).Infoln("ungzip error:", err, r.URL.Path)
}
+ } else if strings.Contains(r.Header.Get("Accept-Encoding"), "zstd") && util.IsZstdContent(n.Data) {
+ w.Header().Set("Content-Encoding", "zstd")
+ } else if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") && util.IsGzippedContent(n.Data) {
+ w.Header().Set("Content-Encoding", "gzip")
} else {
if n.Data, err = util.DecompressData(n.Data); err != nil {
- glog.V(0).Infoln("ungzip error:", err, r.URL.Path)
+ glog.V(0).Infoln("uncompress error:", err, r.URL.Path)
}
}
}
diff --git a/weed/storage/needle/needle_parse_upload.go b/weed/storage/needle/needle_parse_upload.go
index e339c65a4..a2c881f92 100644
--- a/weed/storage/needle/needle_parse_upload.go
+++ b/weed/storage/needle/needle_parse_upload.go
@@ -22,6 +22,7 @@ type ParsedUpload struct {
MimeType string
PairMap map[string]string
IsGzipped bool
+ IsZstd bool
OriginalDataSize int
ModifiedTime uint64
Ttl *TTL
@@ -68,7 +69,7 @@ func ParseUpload(r *http.Request, sizeLimit int64) (pu *ParsedUpload, e error) {
}
} else {
ext := filepath.Base(pu.FileName)
- if shouldGzip, iAmSure := util.IsGzippableFileType(ext, pu.MimeType); pu.MimeType == "" && !iAmSure || shouldGzip && iAmSure {
+ if shouldBeCompressed, iAmSure := util.IsCompressableFileType(ext, pu.MimeType); pu.MimeType == "" && !iAmSure || shouldBeCompressed && iAmSure {
// println("ext", ext, "iAmSure", iAmSure, "shouldGzip", shouldGzip, "mimeType", pu.MimeType)
if compressedData, err := util.GzipData(pu.Data); err == nil {
if len(compressedData)*10 < len(pu.Data)*9 {
@@ -94,6 +95,7 @@ func ParseUpload(r *http.Request, sizeLimit int64) (pu *ParsedUpload, e error) {
func parsePut(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) {
pu.IsGzipped = r.Header.Get("Content-Encoding") == "gzip"
+ pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd"
pu.MimeType = r.Header.Get("Content-Type")
pu.FileName = ""
pu.Data, e = ioutil.ReadAll(io.LimitReader(r.Body, sizeLimit+1))
@@ -187,6 +189,7 @@ func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error
}
pu.IsGzipped = part.Header.Get("Content-Encoding") == "gzip"
+ pu.IsZstd = part.Header.Get("Content-Encoding") == "zstd"
}
return
diff --git a/weed/util/compression.go b/weed/util/compression.go
index 003f06cbf..de6bf0800 100644
--- a/weed/util/compression.go
+++ b/weed/util/compression.go
@@ -8,9 +8,8 @@ import (
"io/ioutil"
"strings"
- "golang.org/x/tools/godoc/util"
-
"github.com/chrislusf/seaweedfs/weed/glog"
+ "github.com/klauspost/compress/zstd"
)
func GzipData(input []byte) ([]byte, error) {
@@ -30,6 +29,9 @@ func DecompressData(input []byte) ([]byte, error) {
if IsGzippedContent(input) {
return ungzipData(input)
}
+ if IsZstdContent(input) {
+ return unzstdData(input)
+ }
return nil, fmt.Errorf("unsupported compression")
}
@@ -44,19 +46,10 @@ func ungzipData(input []byte) ([]byte, error) {
return output, err
}
-/*
-* Default more not to gzip since gzip can be done on client side.
- */
-func IsGzippable(ext, mtype string, data []byte) bool {
-
- shouldBeZipped, iAmSure := IsGzippableFileType(ext, mtype)
- if iAmSure {
- return shouldBeZipped
- }
-
- isMostlyText := util.IsText(data)
+var zstdEncoder, _ = zstd.NewWriter(nil)
- return isMostlyText
+func unzstdData(input []byte) ([]byte, error) {
+ return zstdEncoder.EncodeAll(input, nil), nil
}
func IsGzippedContent(data []byte) bool {
@@ -66,9 +59,16 @@ func IsGzippedContent(data []byte) bool {
return data[0] == 31 && data[1] == 139
}
+func IsZstdContent(data []byte) bool {
+ if len(data) < 4 {
+ return false
+ }
+ return data[0] == 0xFD && data[1] == 0x2F && data[2] == 0xB5 && data[3] == 0x28
+}
+
/*
-* Default more not to gzip since gzip can be done on client side.
- */func IsGzippableFileType(ext, mtype string) (shouldBeZipped, iAmSure bool) {
+* Default not to compressed since compression can be done on client side.
+ */func IsCompressableFileType(ext, mtype string) (shouldBeCompressed, iAmSure bool) {
// text
if strings.HasPrefix(mtype, "text/") {
@@ -86,7 +86,7 @@ func IsGzippedContent(data []byte) bool {
// by file name extension
switch ext {
- case ".zip", ".rar", ".gz", ".bz2", ".xz":
+ case ".zip", ".rar", ".gz", ".bz2", ".xz", ".zst":
return false, true
case ".pdf", ".txt", ".html", ".htm", ".css", ".js", ".json":
return true, true
@@ -98,13 +98,15 @@ func IsGzippedContent(data []byte) bool {
// by mime type
if strings.HasPrefix(mtype, "application/") {
+ if strings.HasSuffix(mtype, "zstd") {
+ return false, true
+ }
if strings.HasSuffix(mtype, "xml") {
return true, true
}
if strings.HasSuffix(mtype, "script") {
return true, true
}
-
}
if strings.HasPrefix(mtype, "audio/") {
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-21 06:33:42 +0000