diff options
| -rw-r--r-- | go.mod | 2 | ||||
| -rw-r--r-- | weed/operation/needle_parse_test.go | 2 | ||||
| -rw-r--r-- | weed/server/volume_server_handlers_read.go | 4 | ||||
| -rw-r--r-- | weed/shell/command_fs_meta_cat.go | 5 | ||||
| -rw-r--r-- | weed/storage/needle/needle_parse_upload.go | 6 | ||||
| -rw-r--r-- | weed/util/compression.go | 29 |
6 files changed, 28 insertions, 20 deletions
@@ -40,7 +40,7 @@ require ( github.com/json-iterator/go v1.1.10 github.com/karlseguin/ccache v2.0.3+incompatible github.com/karlseguin/expect v1.0.1 // indirect - github.com/klauspost/compress v1.10.9 + github.com/klauspost/compress v1.10.9 // indirect github.com/klauspost/cpuid v1.2.1 // indirect github.com/klauspost/crc32 v1.2.0 github.com/klauspost/reedsolomon v1.9.2 diff --git a/weed/operation/needle_parse_test.go b/weed/operation/needle_parse_test.go index 177c620f4..aba89616e 100644 --- a/weed/operation/needle_parse_test.go +++ b/weed/operation/needle_parse_test.go @@ -75,6 +75,7 @@ func TestCreateNeedleFromRequest(t *testing.T) { Upload("http://localhost:8080/389,0f084d17353afda0", "t.txt", false, bytes.NewReader(gzippedData), true, "text/plain", nil, "") } + /* { mc.needleHandling = func(n *needle.Needle, originalSize int, err error) { assert.Equal(t, nil, err, "upload: %v", err) @@ -98,6 +99,7 @@ func TestCreateNeedleFromRequest(t *testing.T) { zstdData, _ := util.ZstdData([]byte(textContent)) Upload("http://localhost:8080/389,0f084d17353afda0", "t.txt", false, bytes.NewReader(zstdData), false, "application/zstd", nil, "") } + */ } diff --git a/weed/server/volume_server_handlers_read.go b/weed/server/volume_server_handlers_read.go index 15fd446e7..5a5495df6 100644 --- a/weed/server/volume_server_handlers_read.go +++ b/weed/server/volume_server_handlers_read.go @@ -159,8 +159,8 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) if n.Data, err = util.DecompressData(n.Data); err != nil { glog.V(0).Infoln("ungzip error:", err, r.URL.Path) } - } else if strings.Contains(r.Header.Get("Accept-Encoding"), "zstd") && util.IsZstdContent(n.Data) { - w.Header().Set("Content-Encoding", "zstd") + // } else if strings.Contains(r.Header.Get("Accept-Encoding"), "zstd") && util.IsZstdContent(n.Data) { + // w.Header().Set("Content-Encoding", "zstd") } else if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") && util.IsGzippedContent(n.Data) { w.Header().Set("Content-Encoding", "gzip") } else { diff --git a/weed/shell/command_fs_meta_cat.go b/weed/shell/command_fs_meta_cat.go index a097a3a4e..e0525defa 100644 --- a/weed/shell/command_fs_meta_cat.go +++ b/weed/shell/command_fs_meta_cat.go @@ -72,8 +72,9 @@ func (c *commandFsMetaCat) Do(args []string, commandEnv *CommandEnv, writer io.W bytes, _ := proto.Marshal(respLookupEntry.Entry) gzippedBytes, _ := util.GzipData(bytes) - zstdBytes, _ := util.ZstdData(bytes) - fmt.Fprintf(writer, "chunks %d meta size: %d gzip:%d zstd:%d\n", len(respLookupEntry.Entry.Chunks), len(bytes), len(gzippedBytes), len(zstdBytes)) + // zstdBytes, _ := util.ZstdData(bytes) + // fmt.Fprintf(writer, "chunks %d meta size: %d gzip:%d zstd:%d\n", len(respLookupEntry.Entry.Chunks), len(bytes), len(gzippedBytes), len(zstdBytes)) + fmt.Fprintf(writer, "chunks %d meta size: %d gzip:%d\n", len(respLookupEntry.Entry.Chunks), len(bytes), len(gzippedBytes)) return nil diff --git a/weed/storage/needle/needle_parse_upload.go b/weed/storage/needle/needle_parse_upload.go index 4d244046e..84e69c73e 100644 --- a/weed/storage/needle/needle_parse_upload.go +++ b/weed/storage/needle/needle_parse_upload.go @@ -23,7 +23,7 @@ type ParsedUpload struct { MimeType string PairMap map[string]string IsGzipped bool - IsZstd bool + // IsZstd bool OriginalDataSize int ModifiedTime uint64 Ttl *TTL @@ -100,7 +100,7 @@ func ParseUpload(r *http.Request, sizeLimit int64) (pu *ParsedUpload, e error) { func parsePut(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) { pu.IsGzipped = r.Header.Get("Content-Encoding") == "gzip" - pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd" + // pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd" pu.MimeType = r.Header.Get("Content-Type") pu.FileName = "" pu.Data, e = ioutil.ReadAll(io.LimitReader(r.Body, sizeLimit+1)) @@ -194,7 +194,7 @@ func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error } pu.IsGzipped = part.Header.Get("Content-Encoding") == "gzip" - pu.IsZstd = part.Header.Get("Content-Encoding") == "zstd" + // pu.IsZstd = part.Header.Get("Content-Encoding") == "zstd" } return diff --git a/weed/util/compression.go b/weed/util/compression.go index cf3ac7c57..33506834b 100644 --- a/weed/util/compression.go +++ b/weed/util/compression.go @@ -9,7 +9,7 @@ import ( "strings" "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/klauspost/compress/zstd" + // "github.com/klauspost/compress/zstd" ) var ( @@ -55,19 +55,16 @@ func GzipData(input []byte) ([]byte, error) { return buf.Bytes(), nil } -var zstdEncoder, _ = zstd.NewWriter(nil) - -func ZstdData(input []byte) ([]byte, error) { - return zstdEncoder.EncodeAll(input, nil), nil -} func DecompressData(input []byte) ([]byte, error) { if IsGzippedContent(input) { return ungzipData(input) } + /* if IsZstdContent(input) { return unzstdData(input) } + */ return input, UnsupportedCompression } @@ -82,12 +79,6 @@ func ungzipData(input []byte) ([]byte, error) { return output, err } -var decoder, _ = zstd.NewReader(nil) - -func unzstdData(input []byte) ([]byte, error) { - return decoder.DecodeAll(input, nil) -} - func IsGzippedContent(data []byte) bool { if len(data) < 2 { return false @@ -95,12 +86,26 @@ func IsGzippedContent(data []byte) bool { return data[0] == 31 && data[1] == 139 } +/* +var zstdEncoder, _ = zstd.NewWriter(nil) + +func ZstdData(input []byte) ([]byte, error) { + return zstdEncoder.EncodeAll(input, nil), nil +} + +var decoder, _ = zstd.NewReader(nil) + +func unzstdData(input []byte) ([]byte, error) { + return decoder.DecodeAll(input, nil) +} + func IsZstdContent(data []byte) bool { if len(data) < 4 { return false } return data[3] == 0xFD && data[2] == 0x2F && data[1] == 0xB5 && data[0] == 0x28 } +*/ /* * Default not to compressed since compression can be done on client side. |