diff options
| -rw-r--r-- | .github/workflows/s3tests.yml | 18 | ||||
| -rw-r--r-- | test/kafka/go.mod | 27 | ||||
| -rw-r--r-- | test/kafka/go.sum | 52 | ||||
| -rw-r--r-- | weed/command/filer.go | 1 | ||||
| -rw-r--r-- | weed/command/s3.go | 3 | ||||
| -rw-r--r-- | weed/command/server.go | 1 | ||||
| -rw-r--r-- | weed/filer/entry.go | 17 | ||||
| -rw-r--r-- | weed/filer/filer.go | 9 | ||||
| -rw-r--r-- | weed/pb/filer_pb/filer_pb_helper.go | 14 | ||||
| -rw-r--r-- | weed/s3api/filer_util.go | 32 | ||||
| -rw-r--r-- | weed/s3api/s3_constants/header.go | 1 | ||||
| -rw-r--r-- | weed/s3api/s3api_bucket_handlers.go | 9 | ||||
| -rw-r--r-- | weed/s3api/s3api_object_handlers.go | 15 | ||||
| -rw-r--r-- | weed/s3api/s3api_object_handlers_list.go | 6 | ||||
| -rw-r--r-- | weed/s3api/s3api_object_handlers_put.go | 4 | ||||
| -rw-r--r-- | weed/s3api/s3api_server.go | 1 | ||||
| -rw-r--r-- | weed/server/filer_server_handlers_write.go | 7 | ||||
| -rw-r--r-- | weed/server/filer_server_handlers_write_autochunk.go | 4 | ||||
| -rw-r--r-- | weed/util/constants_lifecycle_interval_10sec.go | 8 | ||||
| -rw-r--r-- | weed/util/constants_lifecycle_interval_day.go | 8 |
20 files changed, 182 insertions, 55 deletions
diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml index 540247a34..2796bf636 100644 --- a/.github/workflows/s3tests.yml +++ b/.github/workflows/s3tests.yml @@ -54,7 +54,7 @@ jobs: shell: bash run: | cd weed - go install -buildvcs=false + go install -tags s3tests -buildvcs=false set -x # Create clean data directory for this test run export WEED_DATA_DIR="/tmp/seaweedfs-s3tests-$(date +%s)" @@ -64,7 +64,7 @@ jobs: -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \ -volume.max=100 -volume.preStopSeconds=1 \ -master.port=9333 -volume.port=8080 -filer.port=8888 -s3.port=8000 -metricsPort=9324 \ - -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" & + -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.allowDeleteObjectsByTTL=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" & pid=$! # Wait for all SeaweedFS components to be ready @@ -308,7 +308,10 @@ jobs: s3tests/functional/test_s3.py::test_copy_object_ifnonematch_good \ s3tests/functional/test_s3.py::test_lifecycle_set \ s3tests/functional/test_s3.py::test_lifecycle_get \ - s3tests/functional/test_s3.py::test_lifecycle_set_filter + s3tests/functional/test_s3.py::test_lifecycle_set_filter \ + s3tests/functional/test_s3.py::test_lifecycle_expiration \ + s3tests/functional/test_s3.py::test_lifecyclev2_expiration \ + s3tests/functional/test_s3.py::test_lifecycle_expiration_versioning_enabled kill -9 $pid || true # Clean up data directory rm -rf "$WEED_DATA_DIR" || true @@ -791,7 +794,7 @@ jobs: exit 1 fi - go install -tags "sqlite" -buildvcs=false + go install -tags "sqlite s3tests" -buildvcs=false # Create clean data directory for this test run with unique timestamp and process ID export WEED_DATA_DIR="/tmp/seaweedfs-sql-test-$(date +%s)-$$" mkdir -p "$WEED_DATA_DIR" @@ -814,7 +817,7 @@ jobs: -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \ -volume.max=100 -volume.preStopSeconds=1 \ -master.port=9337 -volume.port=8085 -filer.port=8892 -s3.port=8004 -metricsPort=9328 \ - -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" \ + -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.allowDeleteObjectsByTTL=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" \ > /tmp/seaweedfs-sql-server.log 2>&1 & pid=$! @@ -1123,7 +1126,10 @@ jobs: s3tests/functional/test_s3.py::test_copy_object_ifnonematch_good \ s3tests/functional/test_s3.py::test_lifecycle_set \ s3tests/functional/test_s3.py::test_lifecycle_get \ - s3tests/functional/test_s3.py::test_lifecycle_set_filter + s3tests/functional/test_s3.py::test_lifecycle_set_filter \ + s3tests/functional/test_s3.py::test_lifecycle_expiration \ + s3tests/functional/test_s3.py::test_lifecyclev2_expiration \ + s3tests/functional/test_s3.py::test_lifecycle_expiration_versioning_enabled kill -9 $pid || true # Clean up data directory rm -rf "$WEED_DATA_DIR" || true diff --git a/test/kafka/go.mod b/test/kafka/go.mod index 02f6d6999..593b5f3f5 100644 --- a/test/kafka/go.mod +++ b/test/kafka/go.mod @@ -43,25 +43,25 @@ require ( github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aws/aws-sdk-go v1.55.8 // indirect - github.com/aws/aws-sdk-go-v2 v1.39.2 // indirect + github.com/aws/aws-sdk-go-v2 v1.39.4 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 // indirect github.com/aws/aws-sdk-go-v2/config v1.31.3 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.18.10 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.6 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.18.19 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.11 // indirect github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.9 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.9 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.9 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.11 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.9 // indirect github.com/aws/aws-sdk-go-v2/service/s3 v1.88.3 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.29.1 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.38.2 // indirect - github.com/aws/smithy-go v1.23.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.29.8 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.38.9 // indirect + github.com/aws/smithy-go v1.23.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bradenaw/juniper v0.15.3 // indirect github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect @@ -117,6 +117,7 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect github.com/googleapis/gax-go/v2 v2.15.0 // indirect + github.com/gorilla/mux v1.8.1 // indirect github.com/gorilla/schema v1.4.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect @@ -177,7 +178,7 @@ require ( github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.19.1 // indirect github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect - github.com/rclone/rclone v1.71.1 // indirect + github.com/rclone/rclone v1.71.2 // indirect github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect github.com/rdleal/intervalst v1.5.0 // indirect github.com/relvacode/iso8601 v1.6.0 // indirect @@ -202,7 +203,7 @@ require ( github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 // indirect - github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5 // indirect + github.com/t3rm1n4l/go-mega v0.0.0-20250926104142-ccb8d3498e6c // indirect github.com/tklauser/go-sysconf v0.3.15 // indirect github.com/tklauser/numcpus v0.10.0 // indirect github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 // indirect diff --git a/test/kafka/go.sum b/test/kafka/go.sum index 12ba88daa..85f45b85a 100644 --- a/test/kafka/go.sum +++ b/test/kafka/go.sum @@ -102,44 +102,44 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ= github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk= -github.com/aws/aws-sdk-go-v2 v1.39.2 h1:EJLg8IdbzgeD7xgvZ+I8M1e0fL0ptn/M47lianzth0I= -github.com/aws/aws-sdk-go-v2 v1.39.2/go.mod h1:sDioUELIUO9Znk23YVmIk86/9DOpkbyyVb1i/gUNFXY= +github.com/aws/aws-sdk-go-v2 v1.39.4 h1:qTsQKcdQPHnfGYBBs+Btl8QwxJeoWcOcPcixK90mRhg= +github.com/aws/aws-sdk-go-v2 v1.39.4/go.mod h1:yWSxrnioGUZ4WVv9TgMrNUeLV3PFESn/v+6T/Su8gnM= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 h1:i8p8P4diljCr60PpJp6qZXNlgX4m2yQFpYk+9ZT+J4E= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1/go.mod h1:ddqbooRZYNoJ2dsTwOty16rM+/Aqmk/GOXrK8cg7V00= github.com/aws/aws-sdk-go-v2/config v1.31.3 h1:RIb3yr/+PZ18YYNe6MDiG/3jVoJrPmdoCARwNkMGvco= github.com/aws/aws-sdk-go-v2/config v1.31.3/go.mod h1:jjgx1n7x0FAKl6TnakqrpkHWWKcX3xfWtdnIJs5K9CE= -github.com/aws/aws-sdk-go-v2/credentials v1.18.10 h1:xdJnXCouCx8Y0NncgoptztUocIYLKeQxrCgN6x9sdhg= -github.com/aws/aws-sdk-go-v2/credentials v1.18.10/go.mod h1:7tQk08ntj914F/5i9jC4+2HQTAuJirq7m1vZVIhEkWs= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.6 h1:wbjnrrMnKew78/juW7I2BtKQwa1qlf6EjQgS69uYY14= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.6/go.mod h1:AtiqqNrDioJXuUgz3+3T0mBWN7Hro2n9wll2zRUc0ww= +github.com/aws/aws-sdk-go-v2/credentials v1.18.19 h1:Jc1zzwkSY1QbkEcLujwqRTXOdvW8ppND3jRBb/VhBQc= +github.com/aws/aws-sdk-go-v2/credentials v1.18.19/go.mod h1:DIfQ9fAk5H0pGtnqfqkbSIzky82qYnGvh06ASQXXg6A= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.11 h1:X7X4YKb+c0rkI6d4uJ5tEMxXgCZ+jZ/D6mvkno8c8Uw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.11/go.mod h1:EqM6vPZQsZHYvC4Cai35UDg/f5NCEU+vp0WfbVqVcZc= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 h1:0SzCLoPRSK3qSydsaFQWugP+lOBCTPwfcBOm6222+UA= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4/go.mod h1:JAet9FsBHjfdI+TnMBX4ModNNaQHAd3dc/Bk+cNsxeM= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.9 h1:se2vOWGD3dWQUtfn4wEjRQJb1HK1XsNIt825gskZ970= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.9/go.mod h1:hijCGH2VfbZQxqCDN7bwz/4dzxV+hkyhjawAtdPWKZA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.9 h1:6RBnKZLkJM4hQ+kN6E7yWFveOTg8NLPHAkqrs4ZPlTU= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.9/go.mod h1:V9rQKRmK7AWuEsOMnHzKj8WyrIir1yUJbZxDuZLFvXI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11 h1:7AANQZkF3ihM8fbdftpjhken0TP9sBzFbV/Ze/Y4HXA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11/go.mod h1:NTF4QCGkm6fzVwncpkFQqoquQyOolcyXfbpC98urj+c= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11 h1:ShdtWUZT37LCAA4Mw2kJAJtzaszfSHFb5n25sdcv4YE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11/go.mod h1:7bUb2sSr2MZ3M/N+VyETLTQtInemHXb/Fl3s8CLzm0Y= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.9 h1:w9LnHqTq8MEdlnyhV4Bwfizd65lfNCNgdlNC6mM5paE= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.9/go.mod h1:LGEP6EK4nj+bwWNdrvX/FnDTFowdBNwcSPuZu/ouFys= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 h1:oegbebPEMA/1Jny7kvwejowCaHz1FWZAQ94WXFNCyTM= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1/go.mod h1:kemo5Myr9ac0U9JfSjMo9yHLtw+pECEHsFtJ9tqCEI8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 h1:xtuxji5CS0JknaXoACOunXOYOQzgfTvGAc9s2QdCJA4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2/go.mod h1:zxwi0DIR0rcRcgdbl7E2MSOvxDyyXGBlScvBkARFaLQ= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.9 h1:by3nYZLR9l8bUH7kgaMU4dJgYFjyRdFEfORlDpPILB4= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.9/go.mod h1:IWjQYlqw4EX9jw2g3qnEPPWvCE6bS8fKzhMed1OK7c8= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.9 h1:5r34CgVOD4WZudeEKZ9/iKpiT6cM1JyEROpXjOcdWv8= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.9/go.mod h1:dB12CEbNWPbzO2uC6QSWHteqOg4JfBVJOojbAoAUb5I= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.11 h1:GpMf3z2KJa4RnJ0ew3Hac+hRFYLZ9DDjfgXjuW+pB54= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.11/go.mod h1:6MZP3ZI4QQsgUCFTwMZA2V0sEriNQ8k2hmoHF3qjimQ= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.9 h1:wuZ5uW2uhJR63zwNlqWH2W4aL4ZjeJP3o92/W+odDY4= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.9/go.mod h1:/G58M2fGszCrOzvJUkDdY8O9kycodunH4VdT5oBAqls= github.com/aws/aws-sdk-go-v2/service/s3 v1.88.3 h1:P18I4ipbk+b/3dZNq5YYh+Hq6XC0vp5RWkLp1tJldDA= github.com/aws/aws-sdk-go-v2/service/s3 v1.88.3/go.mod h1:Rm3gw2Jov6e6kDuamDvyIlZJDMYk97VeCZ82wz/mVZ0= -github.com/aws/aws-sdk-go-v2/service/sso v1.29.1 h1:8OLZnVJPvjnrxEwHFg9hVUof/P4sibH+Ea4KKuqAGSg= -github.com/aws/aws-sdk-go-v2/service/sso v1.29.1/go.mod h1:27M3BpVi0C02UiQh1w9nsBEit6pLhlaH3NHna6WUbDE= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.2 h1:gKWSTnqudpo8dAxqBqZnDoDWCiEh/40FziUjr/mo6uA= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.2/go.mod h1:x7+rkNmRoEN1U13A6JE2fXne9EWyJy54o3n6d4mGaXQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.38.2 h1:YZPjhyaGzhDQEvsffDEcpycq49nl7fiGcfJTIo8BszI= -github.com/aws/aws-sdk-go-v2/service/sts v1.38.2/go.mod h1:2dIN8qhQfv37BdUYGgEC8Q3tteM3zFxTI1MLO2O3J3c= -github.com/aws/smithy-go v1.23.0 h1:8n6I3gXzWJB2DxBDnfxgBaSX6oe0d/t10qGz7OKqMCE= -github.com/aws/smithy-go v1.23.0/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/aws/aws-sdk-go-v2/service/sso v1.29.8 h1:M5nimZmugcZUO9wG7iVtROxPhiqyZX6ejS1lxlDPbTU= +github.com/aws/aws-sdk-go-v2/service/sso v1.29.8/go.mod h1:mbef/pgKhtKRwrigPPs7SSSKZgytzP8PQ6P6JAAdqyM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.3 h1:S5GuJZpYxE0lKeMHKn+BRTz6PTFpgThyJ+5mYfux7BM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.3/go.mod h1:X4OF+BTd7HIb3L+tc4UlWHVrpgwZZIVENU15pRDVTI0= +github.com/aws/aws-sdk-go-v2/service/sts v1.38.9 h1:Ekml5vGg6sHSZLZJQJagefnVe6PmqC2oiRkBq4F7fU0= +github.com/aws/aws-sdk-go-v2/service/sts v1.38.9/go.mod h1:/e15V+o1zFHWdH3u7lpI3rVBcxszktIKuHKCY2/py+k= +github.com/aws/smithy-go v1.23.1 h1:sLvcH6dfAFwGkHLZ7dGiYF7aK6mg4CgKA/iDKjLDt9M= +github.com/aws/smithy-go v1.23.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bradenaw/juniper v0.15.3 h1:RHIAMEDTpvmzV1wg1jMAHGOoI2oJUSPx3lxRldXnFGo= @@ -540,8 +540,8 @@ github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2eg= github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= -github.com/rclone/rclone v1.71.1 h1:cpODfWTRz5i/WAzXsyW85tzfIKNsd1aq8CE8lUB+0zg= -github.com/rclone/rclone v1.71.1/go.mod h1:NLyX57FrnZ9nVLTY5TRdMmGelrGKbIRYGcgRkNdqqlA= +github.com/rclone/rclone v1.71.2 h1:3Jk5xNPFrZhVABRuN/OPvApuZQddpE2tkhYMuEn1Ud4= +github.com/rclone/rclone v1.71.2/go.mod h1:dCK9FzPDlpkbQJ9M7MmWsmv3X5nibfWe+ogJXu6gSgM= github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 h1:bsUq1dX0N8AOIL7EB/X911+m4EHsnWEHeJ0c+3TTBrg= github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6hamU= @@ -621,8 +621,8 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw= github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= -github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5 h1:Sa+sR8aaAMFwxhXWENEnE6ZpqhZ9d7u1RT2722Rw6hc= -github.com/t3rm1n4l/go-mega v0.0.0-20241213151442-a19cff0ec7b5/go.mod h1:UdZiFUFu6e2WjjtjxivwXWcwc1N/8zgbkBR9QNucUOY= +github.com/t3rm1n4l/go-mega v0.0.0-20250926104142-ccb8d3498e6c h1:BLopNCyqewbE8+BtlIp/Juzu8AJGxz0gHdGADnsblVc= +github.com/t3rm1n4l/go-mega v0.0.0-20250926104142-ccb8d3498e6c/go.mod h1:ykucQyiE9Q2qx1wLlEtZkkNn1IURib/2O+Mvd25i1Fo= github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8= github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4= github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= diff --git a/weed/command/filer.go b/weed/command/filer.go index 053c5a147..1e4b065cd 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -122,6 +122,7 @@ func init() { filerS3Options.auditLogConfig = cmdFiler.Flag.String("s3.auditLogConfig", "", "path to the audit log config file") filerS3Options.allowEmptyFolder = cmdFiler.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders") filerS3Options.allowDeleteBucketNotEmpty = cmdFiler.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket") + filerS3Options.allowDeleteObjectsByTTL = cmdFiler.Flag.Bool("s3.allowDeleteObjectsByTTL", false, "allow deleting all expired entries") filerS3Options.localSocket = cmdFiler.Flag.String("s3.localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock") filerS3Options.tlsCACertificate = cmdFiler.Flag.String("s3.cacert.file", "", "path to the TLS CA certificate file") filerS3Options.tlsVerifyClientCert = cmdFiler.Flag.Bool("s3.tlsVerifyClientCert", false, "whether to verify the client's certificate") diff --git a/weed/command/s3.go b/weed/command/s3.go index fa575b3db..4de1c4fba 100644 --- a/weed/command/s3.go +++ b/weed/command/s3.go @@ -51,6 +51,7 @@ type S3Options struct { metricsHttpIp *string allowEmptyFolder *bool allowDeleteBucketNotEmpty *bool + allowDeleteObjectsByTTL *bool auditLogConfig *string localFilerSocket *string dataCenter *string @@ -80,6 +81,7 @@ func init() { s3StandaloneOptions.metricsHttpIp = cmdS3.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.") s3StandaloneOptions.allowEmptyFolder = cmdS3.Flag.Bool("allowEmptyFolder", true, "allow empty folders") s3StandaloneOptions.allowDeleteBucketNotEmpty = cmdS3.Flag.Bool("allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket") + s3StandaloneOptions.allowDeleteObjectsByTTL = cmdS3.Flag.Bool("allowDeleteObjectsByTTL", false, "allow deleting all expired entries") s3StandaloneOptions.localFilerSocket = cmdS3.Flag.String("localFilerSocket", "", "local filer socket path") s3StandaloneOptions.localSocket = cmdS3.Flag.String("localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock") s3StandaloneOptions.idleTimeout = cmdS3.Flag.Int("idleTimeout", 10, "connection idle seconds") @@ -261,6 +263,7 @@ func (s3opt *S3Options) startS3Server() bool { GrpcDialOption: grpcDialOption, AllowEmptyFolder: *s3opt.allowEmptyFolder, AllowDeleteBucketNotEmpty: *s3opt.allowDeleteBucketNotEmpty, + AllowDeleteObjectsByTTL: *s3opt.allowDeleteObjectsByTTL, LocalFilerSocket: localFilerSocket, DataCenter: *s3opt.dataCenter, FilerGroup: filerGroup, diff --git a/weed/command/server.go b/weed/command/server.go index 8c99d04fd..10d412315 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -165,6 +165,7 @@ func init() { s3Options.auditLogConfig = cmdServer.Flag.String("s3.auditLogConfig", "", "path to the audit log config file") s3Options.allowEmptyFolder = cmdServer.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders") s3Options.allowDeleteBucketNotEmpty = cmdServer.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket") + s3Options.allowDeleteObjectsByTTL = cmdServer.Flag.Bool("s3.allowDeleteObjectsByTTL", false, "allow deleting all expired entries") s3Options.localSocket = cmdServer.Flag.String("s3.localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock") s3Options.bindIp = cmdServer.Flag.String("s3.ip.bind", "", "ip address to bind to. If empty, default to same as -ip.bind option.") s3Options.idleTimeout = cmdServer.Flag.Int("s3.idleTimeout", 10, "connection idle seconds") diff --git a/weed/filer/entry.go b/weed/filer/entry.go index 5bd1a3c56..7a31bc73c 100644 --- a/weed/filer/entry.go +++ b/weed/filer/entry.go @@ -1,6 +1,7 @@ package filer import ( + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "os" "time" @@ -143,3 +144,19 @@ func maxUint64(x, y uint64) uint64 { } return y } + +func (entry *Entry) IsExpireS3Enabled() (found bool) { + if entry.Extended != nil { + _, found = entry.Extended[s3_constants.SeaweedFSExpiresS3] + } + return found +} + +func (entry *Entry) GetS3ExpireTime() (expireTime time.Time) { + if entry.Mtime.IsZero() { + expireTime = entry.Crtime + } else { + expireTime = entry.Mtime + } + return expireTime.Add(time.Duration(entry.TtlSec) * time.Second) +} diff --git a/weed/filer/filer.go b/weed/filer/filer.go index b86ac3c5b..6f498ce58 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -351,13 +351,18 @@ func (f *Filer) FindEntry(ctx context.Context, p util.FullPath) (entry *Entry, e } entry, err = f.Store.FindEntry(ctx, p) if entry != nil && entry.TtlSec > 0 { - if entry.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) { + if entry.IsExpireS3Enabled() { + if entry.GetS3ExpireTime().Before(time.Now()) { + f.Store.DeleteOneEntry(ctx, entry) + return nil, filer_pb.ErrNotFound + } + } else if entry.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) { f.Store.DeleteOneEntry(ctx, entry) return nil, filer_pb.ErrNotFound } } - return + return } func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (expiredCount int64, lastFileName string, err error) { diff --git a/weed/pb/filer_pb/filer_pb_helper.go b/weed/pb/filer_pb/filer_pb_helper.go index b5fd4e1e0..3ce494716 100644 --- a/weed/pb/filer_pb/filer_pb_helper.go +++ b/weed/pb/filer_pb/filer_pb_helper.go @@ -24,6 +24,20 @@ func (entry *Entry) IsDirectoryKeyObject() bool { return entry.IsDirectory && entry.Attributes != nil && entry.Attributes.Mime != "" } +func (entry *Entry) GetExpiryTime() (expiryTime int64) { + expiryTime = entry.Attributes.Mtime + if expiryTime == 0 { + expiryTime = entry.Attributes.Crtime + } + expiryTime += int64(entry.Attributes.TtlSec) + return expiryTime +} + +func (entry *Entry) IsExpired() bool { + return entry != nil && entry.Attributes != nil && entry.Attributes.TtlSec > 0 && + time.Now().Unix() >= entry.GetExpiryTime() +} + func (entry *Entry) FileMode() (fileMode os.FileMode) { if entry != nil && entry.Attributes != nil { fileMode = os.FileMode(entry.Attributes.FileMode) diff --git a/weed/s3api/filer_util.go b/weed/s3api/filer_util.go index 9dd9a684e..e02b945cd 100644 --- a/weed/s3api/filer_util.go +++ b/weed/s3api/filer_util.go @@ -3,6 +3,7 @@ package s3api import ( "context" "fmt" + "math" "strings" "github.com/seaweedfs/seaweedfs/weed/glog" @@ -108,6 +109,37 @@ func (s3a *S3ApiServer) updateEntry(parentDirectoryPath string, newEntry *filer_ return err } +func (s3a *S3ApiServer) updateEntriesTTL(parentDirectoryPath string, ttlSec int32) error { + err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + ctx := context.Background() + err := filer_pb.SeaweedList(ctx, client, parentDirectoryPath, "", func(entry *filer_pb.Entry, isLast bool) error { + if entry.IsDirectory { + return s3a.updateEntriesTTL(fmt.Sprintf("%s/%s", parentDirectoryPath, entry.Name), ttlSec) + } + if entry.Attributes == nil { + entry.Attributes = &filer_pb.FuseAttributes{} + } + if entry.Attributes.TtlSec == ttlSec { + return nil + } + entry.Attributes.TtlSec = ttlSec + err := filer_pb.UpdateEntry(ctx, client, &filer_pb.UpdateEntryRequest{ + Directory: parentDirectoryPath, + Entry: entry, + }) + if err != nil { + return err + } + return nil + }, "", false, math.MaxInt32) + if err != nil { + return err + } + return nil + }) + return err +} + func (s3a *S3ApiServer) getCollectionName(bucket string) string { if s3a.option.FilerGroup != "" { return fmt.Sprintf("%s_%s", s3a.option.FilerGroup, bucket) diff --git a/weed/s3api/s3_constants/header.go b/weed/s3api/s3_constants/header.go index 82a270111..77ed310d9 100644 --- a/weed/s3api/s3_constants/header.go +++ b/weed/s3api/s3_constants/header.go @@ -42,6 +42,7 @@ const ( SeaweedFSIsDirectoryKey = "X-Seaweedfs-Is-Directory-Key" SeaweedFSPartNumber = "X-Seaweedfs-Part-Number" SeaweedFSUploadId = "X-Seaweedfs-Upload-Id" + SeaweedFSExpiresS3 = "X-Seaweedfs-Expires-S3" // S3 ACL headers AmzCannedAcl = "X-Amz-Acl" diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index ead77041e..4a6b88002 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -7,6 +7,7 @@ import ( "encoding/xml" "errors" "fmt" + "github.com/seaweedfs/seaweedfs/weed/util" "math" "net/http" "path" @@ -792,9 +793,9 @@ func (s3a *S3ApiServer) PutBucketLifecycleConfigurationHandler(w http.ResponseWr if rule.Expiration.Days == 0 { continue } - + locationPrefix := fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, bucket, rulePrefix) locConf := &filer_pb.FilerConf_PathConf{ - LocationPrefix: fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, bucket, rulePrefix), + LocationPrefix: locationPrefix, Collection: collectionName, Ttl: fmt.Sprintf("%dd", rule.Expiration.Days), } @@ -806,6 +807,10 @@ func (s3a *S3ApiServer) PutBucketLifecycleConfigurationHandler(w http.ResponseWr s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return } + ttlSec := int32((time.Duration(rule.Expiration.Days) * util.LifeCycleInterval).Seconds()) + if updErr := s3a.updateEntriesTTL(locationPrefix, ttlSec); updErr != nil { + glog.Errorf("PutBucketLifecycleConfigurationHandler update: %s", updErr) + } changed = true } diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 163633e22..a7b6ed736 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -18,6 +18,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util/mem" "github.com/seaweedfs/seaweedfs/weed/glog" @@ -87,6 +88,13 @@ func removeDuplicateSlashes(object string) string { } return result.String() } +func (s3a *S3ApiServer) removeExpiredObject(w http.ResponseWriter, r *http.Request, entry *filer_pb.Entry, bucket, object string) { + target := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object)) + dir, name := target.DirAndName() + if rmErr := s3a.rm(dir, name, true, false); rmErr != nil { + glog.Errorf("delete expired entries %s/%s: %v", dir, name, rmErr) + } +} // checkDirectoryObject checks if the object is a directory object (ends with "/") and if it exists // Returns: (entry, isDirectoryObject, error) @@ -340,6 +348,11 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request) // Add object lock metadata to response headers if present s3a.addObjectLockHeadersToResponse(w, entry) } else { + if s3a.option.AllowDeleteObjectsByTTL && entry.IsExpired() { + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey) + s3a.removeExpiredObject(w, r, entry, bucket, object) + return + } // Handle regular GET (non-versioned) destUrl = s3a.toFilerUrl(bucket, object) } @@ -375,7 +388,6 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request) // Restore the original Range header for SSE processing if sseObject && originalRangeHeader != "" { r.Header.Set("Range", originalRangeHeader) - } // Add SSE metadata headers based on object metadata before SSE processing @@ -603,7 +615,6 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des resp.Body.Close() return } - setUserMetadataKeyToLowercase(resp) responseStatusCode, bytesTransferred := responseFn(resp, w) diff --git a/weed/s3api/s3api_object_handlers_list.go b/weed/s3api/s3api_object_handlers_list.go index 9e6376a0e..f6d24bc95 100644 --- a/weed/s3api/s3api_object_handlers_list.go +++ b/weed/s3api/s3api_object_handlers_list.go @@ -303,6 +303,12 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m } } } + if s3a.option.AllowDeleteObjectsByTTL && entry.IsExpired() { + if delErr := doDeleteEntry(client, dirName, entryName, true, false); delErr != nil { + glog.Errorf("delete expired entries %s/%s: %v", dirName, entryName, delErr) + } + return + } if !delimiterFound { contents = append(contents, newListEntry(entry, "", dirName, entryName, bucketPrefix, fetchOwner, false, false, s3a.iam)) cursor.maxKeys-- diff --git a/weed/s3api/s3api_object_handlers_put.go b/weed/s3api/s3api_object_handlers_put.go index 148df89f6..1609ca510 100644 --- a/weed/s3api/s3api_object_handlers_put.go +++ b/weed/s3api/s3api_object_handlers_put.go @@ -333,7 +333,9 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader proxyReq.Header.Set(s3_constants.SeaweedFSSSES3Key, base64.StdEncoding.EncodeToString(sseS3Metadata)) glog.V(3).Infof("putToFiler: storing SSE-S3 metadata for object %s with keyID %s", uploadUrl, sseS3Key.KeyID) } - + if s3a.option.AllowDeleteObjectsByTTL { + proxyReq.Header.Set(s3_constants.SeaweedFSExpiresS3, "true") + } // ensure that the Authorization header is overriding any previous // Authorization header which might be already present in proxyReq s3a.maybeAddFilerJwtAuthorization(proxyReq, true) diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index e21886c57..baf28b237 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -41,6 +41,7 @@ type S3ApiServerOption struct { GrpcDialOption grpc.DialOption AllowEmptyFolder bool AllowDeleteBucketNotEmpty bool + AllowDeleteObjectsByTTL bool LocalFilerSocket string DataCenter string FilerGroup string diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index 4f1ca05be..31a4f4a41 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -77,7 +77,6 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request, conte if finalDestination := r.Header.Get(s3_constants.SeaweedStorageDestinationHeader); finalDestination != "" { destination = finalDestination } - query := r.URL.Query() so, err := fs.detectStorageOption0(ctx, destination, query.Get("collection"), @@ -99,7 +98,11 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request, conte } return } - + if so.TtlSeconds > 0 { + if S3expiresEnabled := r.Header.Get(s3_constants.SeaweedFSExpiresS3); S3expiresEnabled != "" { + so.TtlSeconds = 0 + } + } if util.FullPath(r.URL.Path).IsLongerFileName(so.MaxFileNameLength) { glog.V(1).InfolnCtx(ctx, "post", r.RequestURI, ": ", "entry name too long") w.WriteHeader(http.StatusRequestURITooLong) diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index d2b3d8b52..308a0421a 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -330,7 +330,9 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa } entry.Extended = SaveAmzMetaData(r, entry.Extended, false) - + if entry.TtlSec > 0 && r.Header.Get(s3_constants.SeaweedFSExpiresS3) == "true" { + entry.Extended[s3_constants.SeaweedFSExpiresS3] = []byte("true") + } for k, v := range r.Header { if len(v) > 0 && len(v[0]) > 0 { if strings.HasPrefix(k, needle.PairNamePrefix) || k == "Cache-Control" || k == "Expires" || k == "Content-Disposition" { diff --git a/weed/util/constants_lifecycle_interval_10sec.go b/weed/util/constants_lifecycle_interval_10sec.go new file mode 100644 index 000000000..60f19c316 --- /dev/null +++ b/weed/util/constants_lifecycle_interval_10sec.go @@ -0,0 +1,8 @@ +//go:build s3tests +// +build s3tests + +package util + +import "time" + +const LifeCycleInterval = 10 * time.Second diff --git a/weed/util/constants_lifecycle_interval_day.go b/weed/util/constants_lifecycle_interval_day.go new file mode 100644 index 000000000..e2465ad5f --- /dev/null +++ b/weed/util/constants_lifecycle_interval_day.go @@ -0,0 +1,8 @@ +//go:build !s3tests +// +build !s3tests + +package util + +import "time" + +const LifeCycleInterval = 24 * time.Hour |