Merge branch 'master' of https://github.com/seaweedfs/seaweedfs

14 files changed, 636 insertions, 100 deletions

diff --git a/.github/workflows/depsreview.yml b/.github/workflows/depsreview.yml
index eeec003d7..32b801eba 100644
--- a/.github/workflows/depsreview.yml
+++ b/.github/workflows/depsreview.yml
@@ -11,4 +11,4 @@ jobs:
       - name: 'Checkout Repository'
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@6c5ccdad469c9f8a2996bfecaec55a631a347034
+        uses: actions/dependency-review-action@9f45b2463b475767b61721ccfef113fef513e6aa
diff --git a/.github/workflows/helm_ci.yml b/.github/workflows/helm_ci.yml
index 161ae6029..942e5a63b 100644
--- a/.github/workflows/helm_ci.yml
+++ b/.github/workflows/helm_ci.yml
@@ -31,7 +31,7 @@ jobs:
           check-latest: true
 
       - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.4.0
+        uses: helm/chart-testing-action@v2.6.1
 
       - name: Run chart-testing (list-changed)
         id: list-changed
diff --git a/go.mod b/go.mod
index d6ed3fba2..68a707bda 100644
--- a/go.mod
+++ b/go.mod
@@ -41,7 +41,6 @@ require (
 	github.com/golang/protobuf v1.5.3
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.1.2
-	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/uuid v1.4.0
 	github.com/google/wire v0.5.0 // indirect
 	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
@@ -120,10 +119,10 @@ require (
 	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/tools v0.14.0
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/api v0.148.0
+	google.golang.org/api v0.149.0
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 // indirect
-	google.golang.org/grpc v1.58.3
+	google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect
+	google.golang.org/grpc v1.59.0
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	modernc.org/b v1.0.0 // indirect
@@ -140,11 +139,11 @@ require (
 
 require (
 	github.com/Jille/raft-grpc-transport v1.4.0
-	github.com/arangodb/go-driver v1.6.0
+	github.com/arangodb/go-driver v1.6.1
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go-v2 v1.21.2
+	github.com/aws/aws-sdk-go-v2 v1.22.1
 	github.com/aws/aws-sdk-go-v2/config v1.18.42
-	github.com/aws/aws-sdk-go-v2/credentials v1.13.43
+	github.com/aws/aws-sdk-go-v2/credentials v1.15.1
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.40.2
 	github.com/fluent/fluent-logger-golang v1.9.0
 	github.com/golang-jwt/jwt/v5 v5.0.0
@@ -155,7 +154,7 @@ require (
 	github.com/orcaman/concurrent-map/v2 v2.0.1
 	github.com/puzpuzpuz/xsync/v2 v2.5.1
 	github.com/rabbitmq/amqp091-go v1.9.0
-	github.com/rclone/rclone v1.64.0
+	github.com/rclone/rclone v1.64.2
 	github.com/rdleal/intervalst v1.2.0
 	github.com/schollz/progressbar/v3 v3.13.1
 	github.com/shirou/gopsutil/v3 v3.23.9
@@ -166,9 +165,9 @@ require (
 )
 
 require (
-	cloud.google.com/go/compute v1.23.0 // indirect
+	cloud.google.com/go/compute v1.23.1 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	cloud.google.com/go/iam v1.1.2 // indirect
+	cloud.google.com/go/iam v1.1.3 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
@@ -189,21 +188,21 @@ require (
 	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.1 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.43 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.6 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sns v1.21.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.24.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 // indirect
-	github.com/aws/smithy-go v1.15.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.17.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.25.0 // indirect
+	github.com/aws/smithy-go v1.16.0 // indirect
 	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/bradenaw/juniper v0.13.1 // indirect
@@ -233,7 +232,7 @@ require (
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
 	github.com/hashicorp/go-hclog v1.5.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
@@ -326,17 +325,17 @@ require (
 	golang.org/x/sync v0.4.0 // indirect
 	golang.org/x/term v0.13.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	lukechampine.com/uint128 v1.2.0 // indirect
-	storj.io/common v0.0.0-20230602145716-d6ea82d58b3d // indirect
+	storj.io/common v0.0.0-20230907123639-5fd0608fd947 // indirect
 	storj.io/drpc v0.0.33 // indirect
-	storj.io/picobuf v0.0.1 // indirect
-	storj.io/uplink v1.11.0 // indirect
+	storj.io/picobuf v0.0.2-0.20230906122608-c4ba17033c6c // indirect
+	storj.io/uplink v1.12.0 // indirect
 )
 
 // replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft
diff --git a/go.sum b/go.sum
index f6dcf46c9..2069549b3 100644
--- a/go.sum
+++ b/go.sum
@@ -26,16 +26,16 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf
 cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
 cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
 cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
-cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY=
-cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
+cloud.google.com/go/compute v1.23.1 h1:V97tBoDaZHb6leicZ1G6DLK2BAaZLJ/7+9BB/En3hR0=
+cloud.google.com/go/compute v1.23.1/go.mod h1:CqB3xpmPKKt3OJpW2ndFIXnA9A4xAy/F3Xp1ixncW78=
 cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
 cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
-cloud.google.com/go/iam v1.1.2 h1:gacbrBdWcoVmGLozRuStX45YKvJtzIjJdAolzUs1sm4=
-cloud.google.com/go/iam v1.1.2/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU=
-cloud.google.com/go/kms v1.15.2 h1:lh6qra6oC4AyWe5fUUUBe/S27k12OHAleOOOw6KakdE=
-cloud.google.com/go/kms v1.15.2/go.mod h1:3hopT4+7ooWRCjc2DxgnpESFxhIraaI2IpAVUEhbT/w=
+cloud.google.com/go/iam v1.1.3 h1:18tKG7DzydKWUnLjonWcJO6wjSCAtzh4GcRKlH/Hrzc=
+cloud.google.com/go/iam v1.1.3/go.mod h1:3khUlaBXfPKKe7huYgEpDn6FtgRyMEqbkvBxrQyY5SE=
+cloud.google.com/go/kms v1.15.3 h1:RYsbxTRmk91ydKCzekI2YjryO4c5Y2M80Zwcs9/D/cI=
+cloud.google.com/go/kms v1.15.3/go.mod h1:AJdXqHxS2GlPyduM99s9iGqi2nwbviBbhV/hdmt4iOQ=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
 cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
@@ -129,8 +129,8 @@ github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEq
 github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
 github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/arangodb/go-driver v1.6.0 h1:NFWj/idqXZxhFVueihMSI2R9NotNIsgvNfM/xmpekb4=
-github.com/arangodb/go-driver v1.6.0/go.mod h1:HQmdGkvNMVBTE3SIPSQ8T/ZddC6iwNsfMR+dDJQxIsI=
+github.com/arangodb/go-driver v1.6.1 h1:bnhrpbA4U1NU13JOWs5sWWYMtQwdjKT0+jkl8dSndyY=
+github.com/arangodb/go-driver v1.6.1/go.mod h1:ywucwwi34GBxxXFWw/ym+7/66//L4K9abxk/sFJro2k=
 github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e h1:Xg+hGrY2LcQBbxd0ZFdbGSyRKTYMZCfBbw/pMJFOk1g=
 github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e/go.mod h1:mq7Shfa/CaixoDxiyAAc5jZ6CVBAyPaNQCGS7mkj4Ho=
 github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
@@ -141,26 +141,29 @@ github.com/aws/aws-sdk-go v1.45.25 h1:c4fLlh5sLdK2DCRTY1z0hyuJZU4ygxX8m1FswL6/nF
 github.com/aws/aws-sdk-go v1.45.25/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/aws/aws-sdk-go-v2 v1.20.0/go.mod h1:uWOr0m0jDsiWw8nnXiqZ+YG6LdvAlGYDLLf2NmHZoy4=
 github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M=
-github.com/aws/aws-sdk-go-v2 v1.21.2 h1:+LXZ0sgo8quN9UOKXXzAWRT3FWd4NxeXWOZom9pE7GA=
 github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM=
+github.com/aws/aws-sdk-go-v2 v1.22.1 h1:sjnni/AuoTXxHitsIdT0FwmqUuNUuHtufcVDErVFT9U=
+github.com/aws/aws-sdk-go-v2 v1.22.1/go.mod h1:Kd0OJtkW3Q0M0lUWGszapWjEvrXDzRW+D21JNsroB+c=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14 h1:Sc82v7tDQ/vdU1WtuSyzZ1I7y/68j//HJ6uozND1IDs=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14/go.mod h1:9NCTOURS8OpxvoAVHq79LK81/zC78hfRWFn+aL0SPcY=
 github.com/aws/aws-sdk-go-v2/config v1.18.42 h1:28jHROB27xZwU0CB88giDSjz7M1Sba3olb5JBGwina8=
 github.com/aws/aws-sdk-go-v2/config v1.18.42/go.mod h1:4AZM3nMMxwlG+eZlxvBKqwVbkDLlnN2a4UGTL6HjaZI=
 github.com/aws/aws-sdk-go-v2/credentials v1.13.40/go.mod h1:VtEHVAAqDWASwdOqj/1huyT6uHbs5s8FUHfDQdky/Rs=
-github.com/aws/aws-sdk-go-v2/credentials v1.13.43 h1:LU8vo40zBlo3R7bAvBVy/ku4nxGEyZe9N8MqAeFTzF8=
-github.com/aws/aws-sdk-go-v2/credentials v1.13.43/go.mod h1:zWJBz1Yf1ZtX5NGax9ZdNjhhI4rgjfgsyk6vTY1yfVg=
+github.com/aws/aws-sdk-go-v2/credentials v1.15.1 h1:hmf6lAm9hk7uLCfapZn/jL05lm6Uwdbn1B0fgjyuf4M=
+github.com/aws/aws-sdk-go-v2/credentials v1.15.1/go.mod h1:QTcHga3ZbQOneJuxmGBOCxiClxmp+TlvmjFexAnJ790=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11/go.mod h1:TEPP4tENqBGO99KwVpV9MlOX4NSrSLP8u3KRy2CDwA8=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 h1:PIktER+hwIG286DqXyvVENjgLTAwGgoeriLDD5C+YlQ=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13/go.mod h1:f/Ib/qYjhV2/qdsf79H3QP/eRE4AkVyEf6sk7XfZ1tg=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.2 h1:gIeH4+o1MN/caGBWjoGQTUTIu94xD6fI5B2+TcwBf70=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.2/go.mod h1:wLyMIo/zPOhQhPXTddpfdkSleyigtFi8iMnC+2m/SK4=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.37/go.mod h1:Pdn4j43v49Kk6+82spO3Tu5gSeQXRsxo56ePPQAvFiA=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 h1:nFBQlGtkbPzp/NjZLuFxRqmT91rLJkgvsEQs68h962Y=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43/go.mod h1:auo+PiyLl0n1l8A0e8RIeR8tOzYPfZZH/JNlrJ8igTQ=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.1 h1:fi1ga6WysOyYb5PAf3Exd6B5GiSNpnZim4h1rhlBqx0=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.1/go.mod h1:V5CY8wNurvPUibTi9mwqUqpiFZ5LnioKWIFUDtIzdI8=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.31/go.mod h1:fTJDMe8LOFYtqiFFFeHA+SVMAwqLhoq0kcInYoLa9Js=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 h1:JRVhO25+r3ar2mKGP7E0LDl8K9/G36gjlqca5iQbaqc=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37/go.mod h1:Qe+2KtKml+FEsQF/DHmDV+xjtche/hwoF75EG4UlHW8=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.1 h1:ZpaV/j48RlPc4AmOZuPv22pJliXjXq8/reL63YzyFnw=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.1/go.mod h1:R8aXraabD2e3qv1csxM14/X9WF4wFMIY0kH4YEtYD5M=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.43 h1:g+qlObJH4Kn4n21g69DjspU0hKTjWtq7naZ9OLCv0ew=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.43/go.mod h1:rzfdUlfA+jdgLDmPKjd3Chq9V7LVLYo1Nz++Wb91aRo=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.6 h1:wmGLw2i8ZTlHLw7a9ULGfQbuccw8uIiNr6sol5bFzc8=
@@ -170,8 +173,9 @@ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15/go.mod h1:
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38 h1:skaFGzv+3kA+v2BPKhuekeb1Hbb105+44r8ASC+q5SE=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38/go.mod h1:epIZoRSSbRIwLPJU5F+OldHhwZPBdpDeQkRdCeY3+00=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 h1:WWZA/I2K4ptBS1kg0kV1JbBtG/umed0vwHRrmcr9z7k=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37/go.mod h1:vBmDnwWXWxNPFRMmG2m/3MKOe+xEcMDo1tanpaWCcck=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.1 h1:2OXw3ppu1XsB6rqKEMV4tnecTjIY3PRV2U6IP6KPJQo=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.1/go.mod h1:FZB4AdakIqW/yERVdGJA6Z9jraax1beXfhBBnK2wwR8=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6 h1:9ulSU5ClouoPIYhDQdg9tpl83d5Yb91PXTKK+17q+ow=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6/go.mod h1:lnc2taBsR9nTlz9meD+lhFZZ9EWY712QHrRflWpTcOA=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.40.2 h1:Ll5/YVCOzRB+gxPqs2uD0R7/MyATC0w85626glSKmp4=
@@ -181,18 +185,19 @@ github.com/aws/aws-sdk-go-v2/service/sns v1.21.1/go.mod h1:laHbYFVzphXdCiT3gitfu
 github.com/aws/aws-sdk-go-v2/service/sqs v1.24.1 h1:KbGaxApdPOT2ZWqJiQY5ApnpNhUGbGTjYiKAidlFwp8=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.24.1/go.mod h1:+phkm4aFvcM4jbsDRGoZ+mD8MMvksHF459Xpy5Z90f0=
 github.com/aws/aws-sdk-go-v2/service/sso v1.14.1/go.mod h1:fIAwKQKBFu90pBxx07BFOMJLpRUGu8VOzLJakeY+0K4=
-github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 h1:JuPGc7IkOP4AaqcZSIcyqLpFSqBWK32rM9+a1g6u73k=
-github.com/aws/aws-sdk-go-v2/service/sso v1.15.2/go.mod h1:gsL4keucRCgW+xA85ALBpRFfdSLH4kHOVSnLMSuBECo=
+github.com/aws/aws-sdk-go-v2/service/sso v1.17.0 h1:I/Oh3IxGPfHXiGnwM54TD6hNr/8TlUrBXAtTyGhR+zw=
+github.com/aws/aws-sdk-go-v2/service/sso v1.17.0/go.mod h1:H6NCMvDBqA+CvIaXzaSqM6LWtzv9BzZrqBOqz+PzRF8=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.1/go.mod h1:yygr8ACQRY2PrEcy3xsUI357stq2AxnFM6DIsR9lij4=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 h1:HFiiRkf1SdaAmV3/BHOFZ9DjFynPHj8G/UIO1lQS+fk=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3/go.mod h1:a7bHA82fyUXOm+ZSWKU6PIoBxrjSprdLoM8xPYvzYVg=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.0 h1:irbXQkfVYIRaewYSXcu4yVk0m2T+JzZd0dkop7FjmO0=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.0/go.mod h1:4wPNCkM22+oRe71oydP66K50ojDUC33XutSMi2pEF/M=
 github.com/aws/aws-sdk-go-v2/service/sts v1.22.0/go.mod h1:VC7JDqsqiwXukYEDjoHh9U0fOJtNWh04FPQz4ct4GGU=
-github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 h1:0BkLfgeDjfZnZ+MhB3ONb01u9pwFYTCZVhlsSSBvlbU=
-github.com/aws/aws-sdk-go-v2/service/sts v1.23.2/go.mod h1:Eows6e1uQEsc4ZaHANmsPRzAKcVDrcmjjWiih2+HUUQ=
+github.com/aws/aws-sdk-go-v2/service/sts v1.25.0 h1:sYIFy8tm1xQwRvVQ4CRuBGXKIg9sHNuG6+3UAQuoujk=
+github.com/aws/aws-sdk-go-v2/service/sts v1.25.0/go.mod h1:S/LOQUeYDfJeJpFCIJDMjy7dwL4aA33HUdVi+i7uH8k=
 github.com/aws/smithy-go v1.14.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
-github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8=
 github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
+github.com/aws/smithy-go v1.16.0 h1:gJZEH/Fqh+RsvlJ1Zt4tVAtV6bKkp3cC+R6FCZMNzik=
+github.com/aws/smithy-go v1.16.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
 github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -376,6 +381,8 @@ github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSM
 github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
 github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
 github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
 github.com/go-zookeeper/zk v1.0.3 h1:7M2kwOsc//9VeeFiPtf+uSJlVpU66x9Ba5+8XK7/TDg=
 github.com/go-zookeeper/zk v1.0.3/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
@@ -495,8 +502,8 @@ github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
 github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/wire v0.5.0 h1:I7ELFeVBr3yfPIcc8+MWvrjk+3VjbcSzoXm3JVa+jD8=
 github.com/google/wire v0.5.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU=
-github.com/googleapis/enterprise-certificate-proxy v0.3.1 h1:SBWmZhjUDRorQxrN0nwzf+AHBxnbFjViHQS4P0yVpmQ=
-github.com/googleapis/enterprise-certificate-proxy v0.3.1/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
+github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs=
+github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas=
@@ -724,6 +731,8 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
+github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
+github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.27.1 h1:rfztXRbg6nv/5f+Raen9RcGoSecHIFgBBLQK3Wdj754=
 github.com/onsi/gomega v1.27.1/go.mod h1:aHX5xOykVYzWOV4WqQy0sy8BQptgukenXpCXfadcIAw=
@@ -807,10 +816,14 @@ github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzX
 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU=
 github.com/puzpuzpuz/xsync/v2 v2.5.1 h1:mVGYAvzDSu52+zaGyNjC+24Xw2bQi3kTr4QJ6N9pIIU=
 github.com/puzpuzpuz/xsync/v2 v2.5.1/go.mod h1:gD2H2krq/w52MfPLE+Uy64TzJDVY7lP2znR9qmR35kU=
+github.com/quic-go/qtls-go1-20 v0.3.2 h1:rRgN3WfnKbyik4dBV8A6girlJVxGand/d+jVKbQq5GI=
+github.com/quic-go/qtls-go1-20 v0.3.2/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
+github.com/quic-go/quic-go v0.38.0 h1:T45lASr5q/TrVwt+jrVccmqHhPL2XuSyoCLVCpfOSLc=
+github.com/quic-go/quic-go v0.38.0/go.mod h1:MPCuRq7KBK2hNcfKj/1iD1BGuN3eAYMeNxp3T42LRUg=
 github.com/rabbitmq/amqp091-go v1.9.0 h1:qrQtyzB4H8BQgEuJwhmVQqVHB9O4+MNDJCCAcpc3Aoo=
 github.com/rabbitmq/amqp091-go v1.9.0/go.mod h1:+jPrT9iY2eLjRaMSRHUhc3z14E/l85kv/f+6luSD3pc=
-github.com/rclone/rclone v1.64.0 h1:5C/7FUEJwtkXwbdrxeioaMj2b9jOqu4CIhQIbnX3144=
-github.com/rclone/rclone v1.64.0/go.mod h1:Et7Df/K2dOZpP2VUNs52b9nK7lS9kc8FeTPtVfMXU7Y=
+github.com/rclone/rclone v1.64.2 h1:vF1FiMw5xRpMKRdODme+EyCiit9spcyMkZcDmZXJG5M=
+github.com/rclone/rclone v1.64.2/go.mod h1:qvuTiatq811sVjaBRJ8RC4Z/CeZxIGdqahz778/YGJo=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rdleal/intervalst v1.2.0 h1:2KrAoGsitJx2NHt9tS+o5aAIi+WqcjbesfcFi1cXxK8=
@@ -1380,8 +1393,8 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513
 google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
 google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
 google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
-google.golang.org/api v0.148.0 h1:HBq4TZlN4/1pNcu0geJZ/Q50vIwIXT532UIMYoo0vOs=
-google.golang.org/api v0.148.0/go.mod h1:8/TBgwaKjfqTdacOJrOv2+2Q6fBDU1uHKK06oGSkxzU=
+google.golang.org/api v0.149.0 h1:b2CqT6kG+zqJIVKRQ3ELJVLN1PwHZ6DJ3dW8yl82rgY=
+google.golang.org/api v0.149.0/go.mod h1:Mwn1B7JTXrzXtnvmzQE2BD6bYZQ8DShKZDZbeN9I7qI=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1431,12 +1444,12 @@ google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
 google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
-google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 h1:SeZZZx0cP0fqUyA+oRzP9k7cSwJlvDFiROO72uwD6i0=
-google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97/go.mod h1:t1VqOqqvce95G3hIDCT5FeO3YUc6Q4Oe24L/+rNMxRk=
-google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 h1:W18sezcAYs+3tDZX4F80yctqa12jcP1PUS2gQu1zTPU=
-google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97/go.mod h1:iargEX0SFPm3xcfMI0d1domjg0ZF4Aa0p2awqyxhvF0=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a h1:a2MQQVoTo96JC9PMGtGBymLp7+/RzpFc2yX/9WfFg1c=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0=
+google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b h1:+YaDE2r2OG8t/z5qmsh7Y+XXwCbvadxxZ0YY6mTdrVA=
+google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:CgAqfJo+Xmu0GwA0411Ht3OU3OntXwsGmrmjI8ioGXI=
+google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b h1:CIC2YMXmIhYw6evmhPxBKJ4fmLbOFtXQN/GV3XOZR8k=
+google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:IBQ646DjkDkvUIsVq/cc03FUFQ9wbZu7yE396YcL870=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -1462,8 +1475,8 @@ google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ5
 google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
 google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
 google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI=
-google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ=
-google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
+google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
+google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
 google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b h1:NuxyvVZoDfHZwYW9LD4GJiF5/nhiSyP4/InTrvw9Ibk=
 google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b/go.mod h1:IBqQ7wSUJ2Ep09a8rMWFsg4fmI2r38zwsq8a0GgxXpM=
 google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1 h1:0emxaJWaG6CfrA9Nbe4aHWbFz5AXw2QPEJP0/f42LCE=
@@ -1556,11 +1569,11 @@ modernc.org/z v1.7.3/go.mod h1:Ipv4tsdxZRbQyLq9Q1M6gdbkxYzdlrciF2Hi/lS7nWE=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-storj.io/common v0.0.0-20230602145716-d6ea82d58b3d h1:AXdJxmg4Jqdz1nmogSrImKOHAU+bn8JCy8lHYnTwP0Y=
-storj.io/common v0.0.0-20230602145716-d6ea82d58b3d/go.mod h1:zu2L8WdpvfIBrCbBTgPsz4qhHSArYSiDgRcV1RLlIF8=
+storj.io/common v0.0.0-20230907123639-5fd0608fd947 h1:X75A5hX1nFjQH8GIvei4T1LNQTLa++bsDKMxXxfPHE8=
+storj.io/common v0.0.0-20230907123639-5fd0608fd947/go.mod h1:FMVOxf2+SgsmfjxwFCM1MZCKwXis4U7l22M/6nIhIas=
 storj.io/drpc v0.0.33 h1:yCGZ26r66ZdMP0IcTYsj7WDAUIIjzXk6DJhbhvt9FHI=
 storj.io/drpc v0.0.33/go.mod h1:vR804UNzhBa49NOJ6HeLjd2H3MakC1j5Gv8bsOQT6N4=
-storj.io/picobuf v0.0.1 h1:ekEvxSQCbEjTVIi/qxj2za13SJyfRE37yE30IBkZeT0=
-storj.io/picobuf v0.0.1/go.mod h1:7ZTAMs6VesgTHbbhFU79oQ9hDaJ+MD4uoFQZ1P4SEz0=
-storj.io/uplink v1.11.0 h1:zGmCcMx1JMRI4NlQi/pN8+z2Jzy7pVVCUDhMVTfboHw=
-storj.io/uplink v1.11.0/go.mod h1:cDlpDWGJykXfYE7NtO1EeArGFy12K5Xj8pV8ufpUCKE=
+storj.io/picobuf v0.0.2-0.20230906122608-c4ba17033c6c h1:or/DtG5uaZpzimL61ahlgAA+MTYn/U3txz4fe+XBFUg=
+storj.io/picobuf v0.0.2-0.20230906122608-c4ba17033c6c/go.mod h1:JCuc3C0gzCJHQ4J6SOx/Yjg+QTpX0D+Fvs5H46FETCk=
+storj.io/uplink v1.12.0 h1:rTODjbKRo/lzz5Hp0isjoRfqDcH7kJg6aujD2M9v9Ro=
+storj.io/uplink v1.12.0/go.mod h1:nMAuoWi5AHio+8NQa33VRzCiRg0B0UhYKuT0a0CdXOg=
diff --git a/test/s3/compatibility/.gitignore b/test/s3/compatibility/.gitignore
index dc3cc5207..f719dbc98 100644
--- a/test/s3/compatibility/.gitignore
+++ b/test/s3/compatibility/.gitignore
@@ -1,2 +1,8 @@
+# Test run scratch data
 /s3-tests
 /tmp
+
+# Test run outputs
+weed.log
+compat.raw.txt
+compat.summary.txt
\ No newline at end of file
diff --git a/test/s3/compatibility/Dockerfile b/test/s3/compatibility/Dockerfile
index b2a1040cb..a3a76b742 100644
--- a/test/s3/compatibility/Dockerfile
+++ b/test/s3/compatibility/Dockerfile
@@ -1,11 +1,21 @@
 # the tests only support python 3.6, not newer
-FROM ubuntu:latest
+#FROM ubuntu:latest
+FROM python:3.6.15-slim-buster
 
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y git-core sudo tzdata
+# Installed required system deps
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y git-core sudo tzdata
+
+# Install python deps
+RUN pip install virtualenv
+
+# Clone Ceph S3 tests
 RUN git clone https://github.com/ceph/s3-tests.git
+
 WORKDIR s3-tests
 
-# we pin a certain commit
+# Pin to a certain commit on ceph/s3-tests
+# https://github.com/ceph/s3-tests/commit/9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e
 RUN git checkout 9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e
 
 RUN ./bootstrap
diff --git a/test/s3/compatibility/Makefile b/test/s3/compatibility/Makefile
new file mode 100644
index 000000000..b6171b10f
--- /dev/null
+++ b/test/s3/compatibility/Makefile
@@ -0,0 +1,9 @@
+.PHONY: all build-image test
+
+all: build-image test
+
+build-image:
+	./prepare.sh
+
+test:
+	./run.sh
diff --git a/test/s3/compatibility/README.md b/test/s3/compatibility/README.md
index de1b6e9ec..5f57cdc9c 100644
--- a/test/s3/compatibility/README.md
+++ b/test/s3/compatibility/README.md
@@ -1,13 +1,245 @@
 # Running S3 Compatibility tests against SeaweedFS
 
-This is using [the tests from CephFS](https://github.com/ceph/s3-tests).
+This is using [the tests from CephFS][s3-tests].
+
+[s3-tests]: https://github.com/ceph/s3-tests
 
 ## Prerequisites
 
-- have Docker installed
+- have [Docker][docker] installed
 - this has been executed on Mac. On Linux, the hostname in `s3tests.conf`  needs to be adjusted.
 
+[docker]: https://docs.docker.com
+
 ## Running tests
 
-- `./prepare.sh` to build the docker image
-- `./run.sh` to execute all tests
+To build the docker image that is used for the tests:
+
+```console
+./prepare.sh
+```
+
+To execute all tests:
+
+```console
+./run.sh
+```
+To see debug output including all commands run by the script:
+
+```console
+DEBUG=y ./run.sh
+```
+
+> [!WARNING]
+>
+> If your output does *not* look like the content in [`results.summary.txt`](./results.summary.txt)
+> and it is full of HTTP level exceptions, there is likely an error contacting the `weed` server from
+> the container that is runnin the S3 compatibility tests.
+>
+> There are at least a couple ways to solve this:
+>
+> - Modify your `docker` setup to ensure `host.docker.internal` is connected to your host running `weed`
+> - Use `--net=host` and modify `host` in `s3tests.conf` to `localhost`
+>
+> The `--net=host` solution is potentially *unsafe*, as the container running [s3-tests][s3-tests] could
+> visit unexpected websites or use the host-passthrough internet access maliciously.
+>
+> If you are OK with the risk of allowing `--net=host`:
+>
+> - Set `host = localhost` in `s3tests.conf`
+> - Set `DOCKER_NET_HOST=y` when running `run.sh`
+
+## Most recent results
+
+See [`results.summary.txt`](./results.summary.txt) for the latest results of compatibility testing (with the caveat that `s3-tests` is pinned to [`ceph/s3-tests` @ 9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e](https://github.com/ceph/s3-tests/commit/9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e)).
+
+The file is reproduced below for ease of access:
+
+```
+/s3-tests/virtualenv/lib/python3.6/site-packages/boto3/compat.py:88: PythonDeprecationWarning: Boto3 will no longer support Python 3.6 starting May 30, 2022. To continue receiving service updates, bug fixes, and security updates please upgrade to Python 3.7 or later. More information can be found here: https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/
+  warnings.warn(warning, PythonDeprecationWarning)
+s3tests_boto3.functional.test_s3.test_bucket_list_return_data ... ERROR
+s3tests_boto3.functional.test_s3.test_object_write_to_nonexist_bucket ... FAIL
+s3tests_boto3.functional.test_s3.test_object_read_not_exist ... ok
+s3tests_boto3.functional.test_s3.test_object_requestid_matches_header_on_error ... FAIL
+s3tests_boto3.functional.test_s3.test_multi_object_delete ... ok
+s3tests_boto3.functional.test_s3.test_multi_objectv2_delete ... ok
+s3tests_boto3.functional.test_s3.test_multi_object_delete_key_limit ... ok
+s3tests_boto3.functional.test_s3.test_multi_objectv2_delete_key_limit ... ok
+s3tests_boto3.functional.test_s3.test_object_head_zero_bytes ... ok
+s3tests_boto3.functional.test_s3.test_object_write_check_etag ... ok
+s3tests_boto3.functional.test_s3.test_object_write_cache_control ... ok
+s3tests_boto3.functional.test_s3.test_object_write_expires ... ok
+s3tests_boto3.functional.test_s3.test_object_write_read_update_read_delete ... ok
+s3tests_boto3.functional.test_s3.test_object_metadata_replaced_on_put ... ok
+s3tests_boto3.functional.test_s3.test_object_write_file ... ok
+s3tests_boto3.functional.test_s3.test_post_object_anonymous_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_no_content_type ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_request_bad_access_key ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_success_code ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_invalid_success_code ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_upload_larger_than_chunk ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_key_from_filename ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_ignored_header ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_case_insensitive_condition_fields ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_escaped_field_values ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_success_redirect_action ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_signature ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_access_key ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_date_format ... ok
+s3tests_boto3.functional.test_s3.test_post_object_no_key_specified ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_signature ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_policy_condition ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_user_specified_header ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_request_missing_policy_specified_field ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_condition_is_case_sensitive ... ok
+s3tests_boto3.functional.test_s3.test_post_object_expires_is_case_sensitive ... ok
+s3tests_boto3.functional.test_s3.test_post_object_expired_policy ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_request_field_value ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_missing_expires_condition ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_conditions_list ... ok
+s3tests_boto3.functional.test_s3.test_post_object_upload_size_limit_exceeded ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_content_length_argument ... ok
+s3tests_boto3.functional.test_s3.test_post_object_invalid_content_length_argument ... ok
+s3tests_boto3.functional.test_s3.test_post_object_upload_size_below_minimum ... ok
+s3tests_boto3.functional.test_s3.test_post_object_empty_conditions ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmatch_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_failed ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_overwrite_existed_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_nonexisted_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_nonexisted_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_overwrite_existed_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_delete_key_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_object_gone ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_response_headers ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_gone ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_not_expired ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_range_zero ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_max_range ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_positive_range ... FAIL
+s3tests_boto3.functional.test_s3.test_object_anon_put ... FAIL
+s3tests_boto3.functional.test_s3.test_object_anon_put_write_access ... ok
+s3tests_boto3.functional.test_s3.test_object_put_authenticated ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_put_authenticated_expired ... FAIL
+s3tests_boto3.functional.test_s3.test_object_acl_canned_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_write ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_writeacp ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_read ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_readacp ... ERROR
+s3tests_boto3.functional.test_s3.test_object_header_acl_grants ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_private ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_private ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicread ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicread ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicreadwrite ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicreadwrite ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_private ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicread ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_private ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicread ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_bucket_create_special_key_names ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_zero_size ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_same_bucket ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_verify_contenttype ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_to_itself ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_to_itself_with_metadata ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_diff_bucket ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_not_owned_bucket ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_not_owned_object_bucket ... ERROR
+s3tests_boto3.functional.test_s3.test_object_copy_canned_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_retaining_metadata ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_replacing_metadata ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_bucket_not_found ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_key_not_found ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_empty ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_small ... ERROR
+s3tests_boto3.functional.test_s3.test_multipart_copy_small ... ok
+s3tests_boto3.functional.test_s3.test_multipart_copy_invalid_range ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_copy_improper_range ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_copy_without_range ... ok
+s3tests_boto3.functional.test_s3.test_multipart_copy_special_names ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload ... ERROR
+s3tests_boto3.functional.test_s3.test_multipart_upload_resend_part ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_size_too_small ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_upload_contents ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_overwrite_existing_object ... ok
+s3tests_boto3.functional.test_s3.test_abort_multipart_upload ... ok
+s3tests_boto3.functional.test_s3.test_abort_multipart_upload_not_found ... ok
+s3tests_boto3.functional.test_s3.test_list_multipart_upload ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_missing_part ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_incorrect_etag ... ok
+s3tests_boto3.functional.test_s3.test_100_continue ... FAIL
+s3tests_boto3.functional.test_s3.test_atomic_read_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_read_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_read_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_conditional_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_conditional_write_1mb ... FAIL
+s3tests_boto3.functional.test_s3.test_atomic_write_bucket_gone ... ok
+s3tests_boto3.functional.test_s3.test_atomic_multipart_upload_write ... ok
+s3tests_boto3.functional.test_s3.test_multipart_resend_first_finishes_last ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_big_request_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_skip_leading_bytes_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_return_trailing_bytes_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_invalid_range ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_empty_object ... ok
+s3tests_boto3.functional.test_s3.test_get_obj_tagging ... ok
+s3tests_boto3.functional.test_s3.test_get_obj_head_tagging ... ok
+s3tests_boto3.functional.test_s3.test_put_max_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_max_kvsize_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_key_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_val_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_modify_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_delete_tags ... ok
+s3tests_boto3.functional.test_s3.test_post_object_tags_anonymous_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_tags_authenticated_request ... FAIL
+s3tests_boto3.functional.test_s3.test_put_obj_with_tags ... ok
+s3tests_boto3.functional.test_s3.test_object_lock_multi_delete_object_with_retention ... ERROR
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_with_bypass ... ok
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_without_bypass ... FAIL
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_compliance ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_good ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_failed ... ok
+s3tests_boto3.functional.test_s3.test_object_read_unreadable ... FAIL
+ERROR
+
+======================================================================
+ERROR: s3tests_boto3.functional.test_s3.test_bucket_list_return_data
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File "/s3-tests/virtualenv/lib/python3.6/site-packages/nose/case.py", line 198, in runTest
+    self.test(*self.arg)
+  File "/s3-tests/s3tests_boto3/functional/test_s3.py", line 1669, in test_bucket_list_return_data
+    'DisplayName': acl_response['Owner']['DisplayName'],
+KeyError: 'Owner'
+-------------------- >> begin captured logging << --------------------
+```
diff --git a/test/s3/compatibility/results.summary.txt b/test/s3/compatibility/results.summary.txt
new file mode 100644
index 000000000..371b9e81a
--- /dev/null
+++ b/test/s3/compatibility/results.summary.txt
@@ -0,0 +1,186 @@
+/s3-tests/virtualenv/lib/python3.6/site-packages/boto3/compat.py:88: PythonDeprecationWarning: Boto3 will no longer support Python 3.6 starting May 30, 2022. To continue receiving service updates, bug fixes, and security updates please upgrade to Python 3.7 or later. More information can be found here: https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/

+  warnings.warn(warning, PythonDeprecationWarning)

+s3tests_boto3.functional.test_s3.test_bucket_list_return_data ... ERROR

+s3tests_boto3.functional.test_s3.test_object_write_to_nonexist_bucket ... FAIL

+s3tests_boto3.functional.test_s3.test_object_read_not_exist ... ok

+s3tests_boto3.functional.test_s3.test_object_requestid_matches_header_on_error ... FAIL

+s3tests_boto3.functional.test_s3.test_multi_object_delete ... ok

+s3tests_boto3.functional.test_s3.test_multi_objectv2_delete ... ok

+s3tests_boto3.functional.test_s3.test_multi_object_delete_key_limit ... ok

+s3tests_boto3.functional.test_s3.test_multi_objectv2_delete_key_limit ... ok

+s3tests_boto3.functional.test_s3.test_object_head_zero_bytes ... ok

+s3tests_boto3.functional.test_s3.test_object_write_check_etag ... ok

+s3tests_boto3.functional.test_s3.test_object_write_cache_control ... ok

+s3tests_boto3.functional.test_s3.test_object_write_expires ... ok

+s3tests_boto3.functional.test_s3.test_object_write_read_update_read_delete ... ok

+s3tests_boto3.functional.test_s3.test_object_metadata_replaced_on_put ... ok

+s3tests_boto3.functional.test_s3.test_object_write_file ... ok

+s3tests_boto3.functional.test_s3.test_post_object_anonymous_request ... FAIL

+s3tests_boto3.functional.test_s3.test_post_object_authenticated_request ... FAIL

+s3tests_boto3.functional.test_s3.test_post_object_authenticated_no_content_type ... FAIL

+s3tests_boto3.functional.test_s3.test_post_object_authenticated_request_bad_access_key ... FAIL

+s3tests_boto3.functional.test_s3.test_post_object_set_success_code ... FAIL

+s3tests_boto3.functional.test_s3.test_post_object_set_invalid_success_code ... FAIL

+s3tests_boto3.functional.test_s3.test_post_object_upload_larger_than_chunk ... FAIL

+s3tests_boto3.functional.test_s3.test_post_object_set_key_from_filename ... FAIL

+s3tests_boto3.functional.test_s3.test_post_object_ignored_header ... FAIL

+s3tests_boto3.functional.test_s3.test_post_object_case_insensitive_condition_fields ... FAIL

+s3tests_boto3.functional.test_s3.test_post_object_escaped_field_values ... FAIL

+s3tests_boto3.functional.test_s3.test_post_object_success_redirect_action ... FAIL

+s3tests_boto3.functional.test_s3.test_post_object_invalid_signature ... FAIL

+s3tests_boto3.functional.test_s3.test_post_object_invalid_access_key ... FAIL

+s3tests_boto3.functional.test_s3.test_post_object_invalid_date_format ... ok

+s3tests_boto3.functional.test_s3.test_post_object_no_key_specified ... ok

+s3tests_boto3.functional.test_s3.test_post_object_missing_signature ... ok

+s3tests_boto3.functional.test_s3.test_post_object_missing_policy_condition ... FAIL

+s3tests_boto3.functional.test_s3.test_post_object_user_specified_header ... FAIL

+s3tests_boto3.functional.test_s3.test_post_object_request_missing_policy_specified_field ... FAIL

+s3tests_boto3.functional.test_s3.test_post_object_condition_is_case_sensitive ... ok

+s3tests_boto3.functional.test_s3.test_post_object_expires_is_case_sensitive ... ok

+s3tests_boto3.functional.test_s3.test_post_object_expired_policy ... FAIL

+s3tests_boto3.functional.test_s3.test_post_object_invalid_request_field_value ... FAIL

+s3tests_boto3.functional.test_s3.test_post_object_missing_expires_condition ... ok

+s3tests_boto3.functional.test_s3.test_post_object_missing_conditions_list ... ok

+s3tests_boto3.functional.test_s3.test_post_object_upload_size_limit_exceeded ... ok

+s3tests_boto3.functional.test_s3.test_post_object_missing_content_length_argument ... ok

+s3tests_boto3.functional.test_s3.test_post_object_invalid_content_length_argument ... ok

+s3tests_boto3.functional.test_s3.test_post_object_upload_size_below_minimum ... ok

+s3tests_boto3.functional.test_s3.test_post_object_empty_conditions ... ok

+s3tests_boto3.functional.test_s3.test_get_object_ifmatch_good ... ok

+s3tests_boto3.functional.test_s3.test_get_object_ifmatch_failed ... ok

+s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_good ... ok

+s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_failed ... ok

+s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_good ... ok

+s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_failed ... ok

+s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_good ... ok

+s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_failed ... ok

+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_good ... ok

+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_failed ... FAIL

+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_overwrite_existed_good ... ok

+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_nonexisted_failed ... FAIL

+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_good ... ok

+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_failed ... FAIL

+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_nonexisted_good ... ok

+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_overwrite_existed_failed ... FAIL

+s3tests_boto3.functional.test_s3.test_object_raw_get ... ok

+s3tests_boto3.functional.test_s3.test_object_raw_get_bucket_gone ... FAIL

+s3tests_boto3.functional.test_s3.test_object_delete_key_bucket_gone ... FAIL

+s3tests_boto3.functional.test_s3.test_object_raw_get_object_gone ... ok

+s3tests_boto3.functional.test_s3.test_object_raw_authenticated ... ok

+s3tests_boto3.functional.test_s3.test_object_raw_response_headers ... ok

+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_acl ... ok

+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_acl ... ok

+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_gone ... FAIL

+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_gone ... ok

+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_not_expired ... ok

+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_range_zero ... FAIL

+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_max_range ... FAIL

+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_positive_range ... FAIL

+s3tests_boto3.functional.test_s3.test_object_anon_put ... FAIL

+s3tests_boto3.functional.test_s3.test_object_anon_put_write_access ... ok

+s3tests_boto3.functional.test_s3.test_object_put_authenticated ... ok

+s3tests_boto3.functional.test_s3.test_object_raw_put_authenticated_expired ... FAIL

+s3tests_boto3.functional.test_s3.test_object_acl_canned_publicreadwrite ... ERROR

+s3tests_boto3.functional.test_s3.test_object_acl ... ERROR

+s3tests_boto3.functional.test_s3.test_object_acl_write ... ERROR

+s3tests_boto3.functional.test_s3.test_object_acl_writeacp ... ERROR

+s3tests_boto3.functional.test_s3.test_object_acl_read ... ERROR

+s3tests_boto3.functional.test_s3.test_object_acl_readacp ... ERROR

+s3tests_boto3.functional.test_s3.test_object_header_acl_grants ... ERROR

+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_private ... FAIL

+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_private ... FAIL

+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicread ... FAIL

+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicread ... FAIL

+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicreadwrite ... FAIL

+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicreadwrite ... FAIL

+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_private ... ERROR

+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicread ... ERROR

+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicreadwrite ... ERROR

+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_private ... ERROR

+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicread ... ERROR

+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicreadwrite ... ERROR

+s3tests_boto3.functional.test_s3.test_bucket_create_special_key_names ... ok

+s3tests_boto3.functional.test_s3.test_object_copy_zero_size ... ok

+s3tests_boto3.functional.test_s3.test_object_copy_same_bucket ... ok

+s3tests_boto3.functional.test_s3.test_object_copy_verify_contenttype ... FAIL

+s3tests_boto3.functional.test_s3.test_object_copy_to_itself ... ok

+s3tests_boto3.functional.test_s3.test_object_copy_to_itself_with_metadata ... ok

+s3tests_boto3.functional.test_s3.test_object_copy_diff_bucket ... ok

+s3tests_boto3.functional.test_s3.test_object_copy_not_owned_bucket ... FAIL

+s3tests_boto3.functional.test_s3.test_object_copy_not_owned_object_bucket ... ERROR

+s3tests_boto3.functional.test_s3.test_object_copy_canned_acl ... ok

+s3tests_boto3.functional.test_s3.test_object_copy_retaining_metadata ... FAIL

+s3tests_boto3.functional.test_s3.test_object_copy_replacing_metadata ... ok

+s3tests_boto3.functional.test_s3.test_object_copy_bucket_not_found ... ok

+s3tests_boto3.functional.test_s3.test_object_copy_key_not_found ... ok

+s3tests_boto3.functional.test_s3.test_multipart_upload_empty ... ok

+s3tests_boto3.functional.test_s3.test_multipart_upload_small ... ERROR

+s3tests_boto3.functional.test_s3.test_multipart_copy_small ... ok

+s3tests_boto3.functional.test_s3.test_multipart_copy_invalid_range ... FAIL

+s3tests_boto3.functional.test_s3.test_multipart_copy_improper_range ... FAIL

+s3tests_boto3.functional.test_s3.test_multipart_copy_without_range ... ok

+s3tests_boto3.functional.test_s3.test_multipart_copy_special_names ... ok

+s3tests_boto3.functional.test_s3.test_multipart_upload ... ERROR

+s3tests_boto3.functional.test_s3.test_multipart_upload_resend_part ... ok

+s3tests_boto3.functional.test_s3.test_multipart_upload_size_too_small ... FAIL

+s3tests_boto3.functional.test_s3.test_multipart_upload_contents ... ok

+s3tests_boto3.functional.test_s3.test_multipart_upload_overwrite_existing_object ... ok

+s3tests_boto3.functional.test_s3.test_abort_multipart_upload ... ok

+s3tests_boto3.functional.test_s3.test_abort_multipart_upload_not_found ... ok

+s3tests_boto3.functional.test_s3.test_list_multipart_upload ... ok

+s3tests_boto3.functional.test_s3.test_multipart_upload_missing_part ... ok

+s3tests_boto3.functional.test_s3.test_multipart_upload_incorrect_etag ... ok

+s3tests_boto3.functional.test_s3.test_100_continue ... FAIL

+s3tests_boto3.functional.test_s3.test_atomic_read_1mb ... ok

+s3tests_boto3.functional.test_s3.test_atomic_read_4mb ... ok

+s3tests_boto3.functional.test_s3.test_atomic_read_8mb ... ok

+s3tests_boto3.functional.test_s3.test_atomic_write_1mb ... ok

+s3tests_boto3.functional.test_s3.test_atomic_write_4mb ... ok

+s3tests_boto3.functional.test_s3.test_atomic_write_8mb ... ok

+s3tests_boto3.functional.test_s3.test_atomic_dual_write_1mb ... ok

+s3tests_boto3.functional.test_s3.test_atomic_dual_write_4mb ... ok

+s3tests_boto3.functional.test_s3.test_atomic_dual_write_8mb ... ok

+s3tests_boto3.functional.test_s3.test_atomic_conditional_write_1mb ... ok

+s3tests_boto3.functional.test_s3.test_atomic_dual_conditional_write_1mb ... FAIL

+s3tests_boto3.functional.test_s3.test_atomic_write_bucket_gone ... ok

+s3tests_boto3.functional.test_s3.test_atomic_multipart_upload_write ... ok

+s3tests_boto3.functional.test_s3.test_multipart_resend_first_finishes_last ... ok

+s3tests_boto3.functional.test_s3.test_ranged_request_response_code ... ok

+s3tests_boto3.functional.test_s3.test_ranged_big_request_response_code ... ok

+s3tests_boto3.functional.test_s3.test_ranged_request_skip_leading_bytes_response_code ... ok

+s3tests_boto3.functional.test_s3.test_ranged_request_return_trailing_bytes_response_code ... ok

+s3tests_boto3.functional.test_s3.test_ranged_request_invalid_range ... ok

+s3tests_boto3.functional.test_s3.test_ranged_request_empty_object ... ok

+s3tests_boto3.functional.test_s3.test_get_obj_tagging ... ok

+s3tests_boto3.functional.test_s3.test_get_obj_head_tagging ... ok

+s3tests_boto3.functional.test_s3.test_put_max_tags ... ok

+s3tests_boto3.functional.test_s3.test_put_excess_tags ... ok

+s3tests_boto3.functional.test_s3.test_put_max_kvsize_tags ... ok

+s3tests_boto3.functional.test_s3.test_put_excess_key_tags ... ok

+s3tests_boto3.functional.test_s3.test_put_excess_val_tags ... ok

+s3tests_boto3.functional.test_s3.test_put_modify_tags ... ok

+s3tests_boto3.functional.test_s3.test_put_delete_tags ... ok

+s3tests_boto3.functional.test_s3.test_post_object_tags_anonymous_request ... FAIL

+s3tests_boto3.functional.test_s3.test_post_object_tags_authenticated_request ... FAIL

+s3tests_boto3.functional.test_s3.test_put_obj_with_tags ... ok

+s3tests_boto3.functional.test_s3.test_object_lock_multi_delete_object_with_retention ... ERROR

+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_with_bypass ... ok

+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_without_bypass ... FAIL

+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_compliance ... FAIL

+s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_good ... ok

+s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_failed ... FAIL

+s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_good ... FAIL

+s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_failed ... ok

+s3tests_boto3.functional.test_s3.test_object_read_unreadable ... FAIL

+ERROR

+

+======================================================================

+ERROR: s3tests_boto3.functional.test_s3.test_bucket_list_return_data

+----------------------------------------------------------------------

+Traceback (most recent call last):

+  File "/s3-tests/virtualenv/lib/python3.6/site-packages/nose/case.py", line 198, in runTest

+    self.test(*self.arg)

+  File "/s3-tests/s3tests_boto3/functional/test_s3.py", line 1669, in test_bucket_list_return_data

+    'DisplayName': acl_response['Owner']['DisplayName'],

+KeyError: 'Owner'

+-------------------- >> begin captured logging << --------------------

diff --git a/test/s3/compatibility/run.sh b/test/s3/compatibility/run.sh
index 990599df5..adfee1366 100755
--- a/test/s3/compatibility/run.sh
+++ b/test/s3/compatibility/run.sh
@@ -1,24 +1,88 @@
 #!/usr/bin/env bash
 
-set -ex
+CONTAINER_NAME=${CONTAINER_NAME:-s3test-instance}
+CONF_FILE=${CONF_FILE:-s3tests.conf}
+WEED_BIN=${WEED_BIN:-../../../weed/weed}
+TEST_RAW_OUTPUT_FILE=${TEST_RAW_OUTPUT_FILE:-compat.raw.txt}
+TEST_PROCESSED_OUTPUT_FILE=${TEST_PROCESSED_OUTPUT_FILE:-compat.summary.txt}
 
+# Set up debugging for this bash script if DEBUG is set
+if [ -n "${DEBUG}" ]; then
+    echo -e "DEBUG set [${DEBUG}], enabling debugging output...";
+    set -ex
+fi
+
+# Reset from possible previous test run
 killall -9 weed || echo "already stopped"
 rm -Rf tmp
 mkdir tmp
-docker stop s3test-instance || echo "already stopped"
+docker stop $CONTAINER_NAME || echo "already stopped"
 
+# Ensure ulimit is set to reasonable value
 ulimit -n 10000
-../../../weed/weed server -filer -s3 -volume.max 0 -master.volumeSizeLimitMB 5 -dir "$(pwd)/tmp" 1>&2>weed.log &
 
+# Start weed w/ filer + s3 in the background
+$WEED_BIN server \
+          -filer \
+          -s3 \
+          -volume.max 0 \
+          -master.volumeSizeLimitMB 5 \
+          -dir "$(pwd)/tmp" \
+          1>&2>weed.log &
+
+# Wait for master to start up
+echo -e "\n[info] waiting for master @ 9333...";
 until curl --output /dev/null --silent --head --fail http://127.0.0.1:9333; do
-    printf '.'
-    sleep 5
+    printf '.';
+    sleep 5;
+done
+sleep 3;
+
+# Wait for s3 to start up
+echo -e "\n[info] waiting for S3 @ 8333...";
+until curl --output /dev/null --silent --fail http://127.0.0.1:8333; do
+    printf '.';
+    sleep 5;
 done
-sleep 3
+sleep 3;
+
+# Determine whether docker net
+DOCKER_NET_HOST_ARGS=""
+if [ -n "${DOCKER_NET_HOST}" ]; then
+    DOCKER_NET_HOST_ARGS="--net=host"
+    echo -e "\n[info] setting docker to het nost"
+fi
+
+echo -e "\n[warn] You may have to run with UNFILTERED=y to disable output filtering, if you get the broken pipe error";
+echo -e "\n[info] running tests with unfiltered output...";
+docker run \
+       --name $CONTAINER_NAME \
+       --rm \
+       ${DOCKER_NET_HOST_ARGS} \
+       -e S3TEST_CONF=$CONF_FILE \
+       -v "$(pwd)"/$CONF_FILE:/s3-tests/s3tests.conf \
+       -it \
+       s3tests \
+       ./virtualenv/bin/nosetests \
+       s3tests_boto3/functional/test_s3.py \
+       -v \
+       -a 'resource=object,!bucket-policy,!versioning,!encryption' \
+    | tee ${TEST_RAW_OUTPUT_FILE}
+
+# If the summary logs are present, process them
+if [ -f "${TEST_RAW_OUTPUT_FILE}" ]; then
+    cat ${TEST_RAW_OUTPUT_FILE} | sed -n -e '/botocore.hooks/!p;//q' | tee ${TEST_PROCESSED_OUTPUT_FILE}
+    echo -e "\n[info] ✅ Successfully wrote processed output @ [${TEST_PROCESSED_OUTPUT_FILE}]";
+    if [ -z "${TEST_KEEP_RAW_OUTPUT}" ]; then
+        echo -e "\n[info] removing test raw output file @ [${TEST_RAW_OUTPUT_FILE}] (to disable this, set TEST_KEEP_RAW_OUTPUT=y)...";
+        rm -rf ${TEST_RAW_OUTPUT_FILE};
+    fi
+else
+    echo -e "\n[warn] failed to find raw output @ [${TEST_RAW_OUTPUT_FILE}]";
+fi
 
-rm -Rf logs-full.txt logs-summary.txt
-# docker run --name s3test-instance --rm -e S3TEST_CONF=s3tests.conf -v `pwd`/s3tests.conf:/s3-tests/s3tests.conf -it s3tests    ./virtualenv/bin/nosetests s3tests_boto3/functional/test_s3.py:test_get_obj_tagging -v  -a 'resource=object,!bucket-policy,!versioning,!encryption'
-docker run --name s3test-instance --rm -e S3TEST_CONF=s3tests.conf -v "$(pwd)"/s3tests.conf:/s3-tests/s3tests.conf -it s3tests    ./virtualenv/bin/nosetests s3tests_boto3/functional/test_s3.py -v  -a 'resource=object,!bucket-policy,!versioning,!encryption' | sed -n -e '/botocore.hooks/!p;//q' | tee logs-summary.txt
+echo -e "\n[info] stopping [${CONTAINER_NAME}] container...";
+docker stop $CONTAINER_NAME || echo "[info] already stopped";
 
-docker stop s3test-instance || echo "already stopped"
-killall -9 weed
+echo -e "\n[info] stopping seaweedfs processes (all, via kill -9)...";
+killall -9 weed;
diff --git a/test/s3/compatibility/s3tests.conf b/test/s3/compatibility/s3tests.conf
index 5adb61791..191168b32 100644
--- a/test/s3/compatibility/s3tests.conf
+++ b/test/s3/compatibility/s3tests.conf
@@ -4,6 +4,14 @@
 # host set for rgw in vstart.sh
 host = host.docker.internal
 
+## NOTE: if running docker --net=host (e.x. `DOCKER_NET_HOST=y ./run.sh`)
+## then use 'localhost', so the container can more easily access your *local* weed instance
+##
+## This is *unsafe* -- it is possible that this script will do malicious
+## things with host-level access. only run this if you trust the programmatic content of
+## ceph/s3-tests (https://github.com/ceph/s3-tests) which will be using this file
+#host = localhost
+
 # port set for rgw in vstart.sh
 port = 8333
 
diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go
index dcb92aa6b..9523eee56 100644
--- a/weed/server/volume_server_handlers.go
+++ b/weed/server/volume_server_handlers.go
@@ -36,13 +36,14 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
 		w.Header().Set("Access-Control-Allow-Origin", "*")
 		w.Header().Set("Access-Control-Allow-Credentials", "true")
 	}
-	stats.VolumeServerRequestCounter.WithLabelValues(r.Method).Inc()
 	start := time.Now()
-	defer func(start time.Time) {
-		stats.VolumeServerRequestHistogram.WithLabelValues(r.Method).Observe(time.Since(start).Seconds())
-	}(start)
+	requestMethod := r.Method
+	defer func(start time.Time, method *string) {
+		stats.VolumeServerRequestCounter.WithLabelValues(*method).Inc()
+		stats.VolumeServerRequestHistogram.WithLabelValues(*method).Observe(time.Since(start).Seconds())
+	}(start, &requestMethod)
 	switch r.Method {
-	case "GET", "HEAD":
+	case http.MethodGet, http.MethodHead:
 		stats.ReadRequest()
 		vs.inFlightDownloadDataLimitCond.L.Lock()
 		inFlightDownloadSize := atomic.LoadInt64(&vs.inFlightDownloadDataSize)
@@ -61,10 +62,12 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
 		}
 		vs.inFlightDownloadDataLimitCond.L.Unlock()
 		vs.GetOrHeadHandler(w, r)
-	case "DELETE":
+	case http.MethodDelete:
+		stats.VolumeServerRequestCounter.WithLabelValues(r.Method).Inc()
 		stats.DeleteRequest()
 		vs.guard.WhiteList(vs.DeleteHandler)(w, r)
-	case "PUT", "POST":
+	case http.MethodPut, http.MethodPost:
+		stats.VolumeServerRequestCounter.WithLabelValues(r.Method).Inc()
 		contentLength := getContentLength(r)
 		// exclude the replication from the concurrentUploadLimitMB
 		if r.URL.Query().Get("type") != "replicate" && vs.concurrentUploadLimit != 0 {
@@ -98,10 +101,13 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
 		stats.WriteRequest()
 		vs.guard.WhiteList(vs.PostHandler)(w, r)
 
-	case "OPTIONS":
+	case http.MethodOptions:
 		stats.ReadRequest()
 		w.Header().Add("Access-Control-Allow-Methods", "PUT, POST, GET, DELETE, OPTIONS")
 		w.Header().Add("Access-Control-Allow-Headers", "*")
+	default:
+		requestMethod = "INVALID"
+		writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("unsupported method %s", r.Method))
 	}
 }
 
@@ -124,7 +130,7 @@ func (vs *VolumeServer) publicReadOnlyHandler(w http.ResponseWriter, r *http.Req
 		w.Header().Set("Access-Control-Allow-Credentials", "true")
 	}
 	switch r.Method {
-	case "GET", "HEAD":
+	case http.MethodGet, http.MethodHead:
 		stats.ReadRequest()
 		vs.inFlightDownloadDataLimitCond.L.Lock()
 		inFlightDownloadSize := atomic.LoadInt64(&vs.inFlightDownloadDataSize)
@@ -135,7 +141,7 @@ func (vs *VolumeServer) publicReadOnlyHandler(w http.ResponseWriter, r *http.Req
 		}
 		vs.inFlightDownloadDataLimitCond.L.Unlock()
 		vs.GetOrHeadHandler(w, r)
-	case "OPTIONS":
+	case http.MethodOptions:
 		stats.ReadRequest()
 		w.Header().Add("Access-Control-Allow-Methods", "GET, OPTIONS")
 		w.Header().Add("Access-Control-Allow-Headers", "*")
diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go
index 945c68279..35c554839 100644
--- a/weed/server/webdav_server.go
+++ b/weed/server/webdav_server.go
@@ -568,6 +568,9 @@ func (f *WebDavFile) Readdir(count int) (ret []os.FileInfo, err error) {
 		ret = append(ret, &fi)
 		return nil
 	})
+	if err != nil {
+		return nil, err
+	}
 
 	old := f.off
 	if old >= int64(len(ret)) {
diff --git a/weed/shell/command_volume_fix_replication.go b/weed/shell/command_volume_fix_replication.go
index 43db0ff3b..9b6a64e6f 100644
--- a/weed/shell/command_volume_fix_replication.go
+++ b/weed/shell/command_volume_fix_replication.go
@@ -572,12 +572,12 @@ func isMisplaced(replicas []*VolumeReplica, replicaPlacement *super_block.Replic
 
 	for i := 0; i < len(replicas); i++ {
 		others := otherThan(replicas, i)
-		if satisfyReplicaPlacement(replicaPlacement, others, *replicas[i].location) {
-			return false
+		if !satisfyReplicaPlacement(replicaPlacement, others, *replicas[i].location) {
+			return true
 		}
 	}
 
-	return true
+	return false
 
 }