implement s3 streaming-unsigned-payload-trailer (#6539)

* implement s3 streaming-unsigned-payload-trailer

* chore: remove print

1 files changed, 114 insertions, 25 deletions

diff --git a/weed/s3api/chunked_reader_v4_test.go b/weed/s3api/chunked_reader_v4_test.go
index 16d4a3db3..786df3465 100644
--- a/weed/s3api/chunked_reader_v4_test.go
+++ b/weed/s3api/chunked_reader_v4_test.go
@@ -2,19 +2,20 @@ package s3api
 
 import (
 	"bytes"
+	"encoding/base64"
+	"fmt"
 	"io"
 	"net/http"
 	"strings"
 	"sync"
 	"testing"
 
+	"hash/crc32"
+
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 	"github.com/stretchr/testify/assert"
 )
 
-// This test will implement the following scenario:
-// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#example-signature-calculations-streaming
-
 const (
 	defaultTimestamp       = "20130524T000000Z"
 	defaultBucketName      = "examplebucket"
@@ -23,19 +24,26 @@ const (
 	defaultRegion          = "us-east-1"
 )
 
-func generatePayload() string {
+func generatestreamingAws4HmacSha256Payload() string {
+	// This test will implement the following scenario:
+	// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#example-signature-calculations-streaming
+
 	chunk1 := "10000;chunk-signature=ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648\r\n" +
 		strings.Repeat("a", 65536) + "\r\n"
 	chunk2 := "400;chunk-signature=0055627c9e194cb4542bae2aa5492e3c1575bbb81b612b7d234b86a503ef5497\r\n" +
 		strings.Repeat("a", 1024) + "\r\n"
-	chunk3 := "0;chunk-signature=b6c6ea8a5354eaf15b3cb7646744f4275b71ea724fed81ceb9323e279d449df9\r\n\r\n"
+	chunk3 := "0;chunk-signature=b6c6ea8a5354eaf15b3cb7646744f4275b71ea724fed81ceb9323e279d449df9\r\n" +
+		"\r\n" // The last chunk is empty
 
 	payload := chunk1 + chunk2 + chunk3
 	return payload
 }
 
-func NewRequest() (*http.Request, error) {
-	payload := generatePayload()
+func NewRequeststreamingAws4HmacSha256Payload() (*http.Request, error) {
+	// This test will implement the following scenario:
+	// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#example-signature-calculations-streaming
+
+	payload := generatestreamingAws4HmacSha256Payload()
 	req, err := http.NewRequest("PUT", "http://s3.amazonaws.com/examplebucket/chunkObject.txt", bytes.NewReader([]byte(payload)))
 	if err != nil {
 		return nil, err
@@ -53,13 +61,109 @@ func NewRequest() (*http.Request, error) {
 	return req, nil
 }
 
-func TestNewSignV4ChunkedReader(t *testing.T) {
-	req, err := NewRequest()
+func TestNewSignV4ChunkedReaderstreamingAws4HmacSha256Payload(t *testing.T) {
+	// This test will implement the following scenario:
+	// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#example-signature-calculations-streaming
+	req, err := NewRequeststreamingAws4HmacSha256Payload()
 	if err != nil {
 		t.Fatalf("Failed to create request: %v", err)
 	}
+	iam := setupIam()
+
+	// The expected payload a long string of 'a's
+	expectedPayload := strings.Repeat("a", 66560)
+
+	runWithRequest(iam, req, t, expectedPayload)
+}
+
+func generateStreamingUnsignedPayloadTrailerPayload(includeFinalCRLF bool) string {
+	// This test will implement the following scenario:
+	// https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
+
+	chunk1 := "2000\r\n" + strings.Repeat("a", 8192) + "\r\n"
+	chunk2 := "2000\r\n" + strings.Repeat("a", 8192) + "\r\n"
+	chunk3 := "400\r\n" + strings.Repeat("a", 1024) + "\r\n"
+
+	chunk4 := "0\r\n" /* the last chunk is empty */
+
+	if includeFinalCRLF {
+		// Some clients omit the final CRLF, so we need to test that case as well
+		chunk4 += "\r\n"
+	}
+
+	data := strings.Repeat("a", 17408)
+	writer := crc32.NewIEEE()
+	_, err := writer.Write([]byte(data))
+
+	if err != nil {
+		fmt.Println("Error:", err)
+	}
+	checksum := writer.Sum(nil)
+	base64EncodedChecksum := base64.StdEncoding.EncodeToString(checksum)
+	trailer := "x-amz-checksum-crc32:" + base64EncodedChecksum + "\n\r\n\r\n\r\n"
 
+	payload := chunk1 + chunk2 + chunk3 + chunk4 + trailer
+	return payload
+}
+
+func NewRequestStreamingUnsignedPayloadTrailer(includeFinalCRLF bool) (*http.Request, error) {
+	// This test will implement the following scenario:
+	// https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
+
+	payload := generateStreamingUnsignedPayloadTrailerPayload(includeFinalCRLF)
+	req, err := http.NewRequest("PUT", "http://amzn-s3-demo-bucket/Key+", bytes.NewReader([]byte(payload)))
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("Host", "amzn-s3-demo-bucket")
+	req.Header.Set("x-amz-date", defaultTimestamp)
+	req.Header.Set("Content-Encoding", "aws-chunked")
+	req.Header.Set("x-amz-decoded-content-length", "17408")
+	req.Header.Set("x-amz-content-sha256", "STREAMING-UNSIGNED-PAYLOAD-TRAILER")
+	req.Header.Set("x-amz-trailer", "x-amz-checksum-crc32")
+
+	return req, nil
+}
+
+func TestNewSignV4ChunkedReaderStreamingUnsignedPayloadTrailer(t *testing.T) {
+	// This test will implement the following scenario:
+	// https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
+	iam := setupIam()
+
+	req, err := NewRequestStreamingUnsignedPayloadTrailer(true)
+	if err != nil {
+		t.Fatalf("Failed to create request: %v", err)
+	}
+	// The expected payload a long string of 'a's
+	expectedPayload := strings.Repeat("a", 17408)
+
+	runWithRequest(iam, req, t, expectedPayload)
+
+	req, err = NewRequestStreamingUnsignedPayloadTrailer(false)
+	if err != nil {
+		t.Fatalf("Failed to create request: %v", err)
+	}
+	runWithRequest(iam, req, t, expectedPayload)
+}
+
+func runWithRequest(iam IdentityAccessManagement, req *http.Request, t *testing.T, expectedPayload string) {
+	reader, errCode := iam.newChunkedReader(req)
+	assert.NotNil(t, reader)
+	assert.Equal(t, s3err.ErrNone, errCode)
+
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		t.Fatalf("Failed to read data: %v", err)
+	}
+
+	assert.Equal(t, expectedPayload, string(data))
+}
+
+func setupIam() IdentityAccessManagement {
 	// Create an IdentityAccessManagement instance
+	// Add default access keys and secrets
+
 	iam := IdentityAccessManagement{
 		identities:        []*Identity{},
 		accessKeyIdent:    map[string]*Identity{},
@@ -72,7 +176,6 @@ func TestNewSignV4ChunkedReader(t *testing.T) {
 		isAuthEnabled:     false,
 	}
 
-	// Add default access keys and secrets
 	iam.identities = append(iam.identities, &Identity{
 		Name: "default",
 		Credentials: []*Credential{
@@ -89,19 +192,5 @@ func TestNewSignV4ChunkedReader(t *testing.T) {
 	})
 
 	iam.accessKeyIdent[defaultAccessKeyId] = iam.identities[0]
-
-	// Call newSignV4ChunkedReader
-	reader, errCode := iam.newSignV4ChunkedReader(req)
-	assert.NotNil(t, reader)
-	assert.Equal(t, s3err.ErrNone, errCode)
-
-	data, err := io.ReadAll(reader)
-	if err != nil {
-		t.Fatalf("Failed to read data: %v", err)
-	}
-
-	// The expected payload a long string of 'a's
-	expectedPayload := strings.Repeat("a", 66560)
-	assert.Equal(t, expectedPayload, string(data))
-
+	return iam
 }