diff options
32 files changed, 301 insertions, 116 deletions
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 35ab5f3e7..6b5d5eb3a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -22,7 +22,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 # Override language selection by uncommenting this and choosing your languages with: languages: go @@ -30,7 +30,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below). - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 # âšī¸ Command-line programs to run using the OS shell. # đ See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun @@ -44,4 +44,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml index 29b47afef..a21842048 100644 --- a/.github/workflows/container_dev.yml +++ b/.github/workflows/container_dev.yml @@ -20,7 +20,7 @@ jobs: - name: Docker meta id: docker_meta - uses: docker/metadata-action@31cebacef4805868f9ce9a0cb03ee36c32df2ac4 # v3 + uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3 with: images: | chrislusf/seaweedfs diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml index e6702faea..8c5e2478b 100644 --- a/.github/workflows/container_latest.yml +++ b/.github/workflows/container_latest.yml @@ -21,7 +21,7 @@ jobs: - name: Docker meta id: docker_meta - uses: docker/metadata-action@31cebacef4805868f9ce9a0cb03ee36c32df2ac4 # v3 + uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3 with: images: | chrislusf/seaweedfs diff --git a/.github/workflows/container_release1.yml b/.github/workflows/container_release1.yml index 2553341e1..7c93f4211 100644 --- a/.github/workflows/container_release1.yml +++ b/.github/workflows/container_release1.yml @@ -20,7 +20,7 @@ jobs: - name: Docker meta id: docker_meta - uses: docker/metadata-action@31cebacef4805868f9ce9a0cb03ee36c32df2ac4 # v3 + uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3 with: images: | chrislusf/seaweedfs diff --git a/.github/workflows/container_release2.yml b/.github/workflows/container_release2.yml index 19359e28f..4198ddb29 100644 --- a/.github/workflows/container_release2.yml +++ b/.github/workflows/container_release2.yml @@ -21,7 +21,7 @@ jobs: - name: Docker meta id: docker_meta - uses: docker/metadata-action@31cebacef4805868f9ce9a0cb03ee36c32df2ac4 # v3 + uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3 with: images: | chrislusf/seaweedfs diff --git a/.github/workflows/container_release3.yml b/.github/workflows/container_release3.yml index 6929b3d03..80a84edfc 100644 --- a/.github/workflows/container_release3.yml +++ b/.github/workflows/container_release3.yml @@ -21,7 +21,7 @@ jobs: - name: Docker meta id: docker_meta - uses: docker/metadata-action@31cebacef4805868f9ce9a0cb03ee36c32df2ac4 # v3 + uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3 with: images: | chrislusf/seaweedfs diff --git a/.github/workflows/container_release4.yml b/.github/workflows/container_release4.yml index af69d6dfd..37146793f 100644 --- a/.github/workflows/container_release4.yml +++ b/.github/workflows/container_release4.yml @@ -20,7 +20,7 @@ jobs: - name: Docker meta id: docker_meta - uses: docker/metadata-action@31cebacef4805868f9ce9a0cb03ee36c32df2ac4 # v3 + uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3 with: images: | chrislusf/seaweedfs diff --git a/.github/workflows/container_release5.yml b/.github/workflows/container_release5.yml index 632662c38..1e0ee550c 100644 --- a/.github/workflows/container_release5.yml +++ b/.github/workflows/container_release5.yml @@ -20,7 +20,7 @@ jobs: - name: Docker meta id: docker_meta - uses: docker/metadata-action@31cebacef4805868f9ce9a0cb03ee36c32df2ac4 # v3 + uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3 with: images: | chrislusf/seaweedfs diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 81fd69654..0a67ef44b 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -94,7 +94,7 @@ jobs: - name: Archive logs if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: output-logs path: docker/output.log diff --git a/.github/workflows/helm_ci.yml b/.github/workflows/helm_ci.yml index 5f1721ed4..86f6b83d0 100644 --- a/.github/workflows/helm_ci.yml +++ b/.github/workflows/helm_ci.yml @@ -36,16 +36,16 @@ jobs: - name: Run chart-testing (list-changed) id: list-changed run: | - changed=$(ct list-changed --chart-dirs k8s/charts) + changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }} --chart-dirs k8s/charts) if [[ -n "$changed" ]]; then echo "::set-output name=changed::true" fi - name: Run chart-testing (lint) - run: ct lint --all --validate-maintainers=false --chart-dirs k8s/charts + run: ct lint --target-branch ${{ github.event.repository.default_branch }} --all --validate-maintainers=false --chart-dirs k8s/charts - name: Create kind cluster uses: helm/kind-action@v1.8.0 - name: Run chart-testing (install) - run: ct install --all --chart-dirs k8s/charts
\ No newline at end of file + run: ct install --target-branch ${{ github.event.repository.default_branch }} --all --chart-dirs k8s/charts @@ -5,7 +5,7 @@ go 1.21 require ( cloud.google.com/go v0.110.10 // indirect cloud.google.com/go/pubsub v1.33.0 - cloud.google.com/go/storage v1.35.1 + cloud.google.com/go/storage v1.36.0 github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-storage-blob-go v0.15.0 github.com/Shopify/sarama v1.38.1 @@ -61,7 +61,7 @@ require ( github.com/klauspost/reedsolomon v1.11.8 github.com/kurin/blazer v0.5.3 github.com/lib/pq v1.10.9 - github.com/linxGnu/grocksdb v1.8.0 + github.com/linxGnu/grocksdb v1.8.10 github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-ieproxy v0.0.11 // indirect @@ -109,7 +109,7 @@ require ( gocloud.dev v0.35.0 gocloud.dev/pubsub/natspubsub v0.34.0 gocloud.dev/pubsub/rabbitpubsub v0.35.0 - golang.org/x/crypto v0.16.0 // indirect + golang.org/x/crypto v0.17.0 // indirect golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/image v0.14.0 golang.org/x/net v0.19.0 @@ -128,7 +128,7 @@ require ( modernc.org/cc/v3 v3.40.0 // indirect modernc.org/ccgo/v3 v3.16.13 // indirect modernc.org/libc v1.29.0 // indirect - modernc.org/mathutil v1.6.0 // indirect + modernc.org/mathutil v1.6.0 modernc.org/memory v1.7.2 // indirect modernc.org/opt v0.1.3 // indirect modernc.org/sqlite v1.27.0 @@ -140,10 +140,10 @@ require ( github.com/Jille/raft-grpc-transport v1.5.0 github.com/arangodb/go-driver v1.6.1 github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go-v2 v1.23.1 - github.com/aws/aws-sdk-go-v2/config v1.25.5 - github.com/aws/aws-sdk-go-v2/credentials v1.16.4 - github.com/aws/aws-sdk-go-v2/service/s3 v1.44.0 + github.com/aws/aws-sdk-go-v2 v1.24.0 + github.com/aws/aws-sdk-go-v2/config v1.26.1 + github.com/aws/aws-sdk-go-v2/credentials v1.16.12 + github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5 github.com/fluent/fluent-logger-golang v1.9.0 github.com/golang-jwt/jwt/v5 v5.1.0 github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50 @@ -187,22 +187,22 @@ require ( github.com/abbot/go-http-auth v0.4.0 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.5 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.4 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 // indirect github.com/aws/aws-sdk-go-v2/service/sns v1.25.3 // indirect github.com/aws/aws-sdk-go-v2/service/sqs v1.28.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.17.3 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.25.4 // indirect - github.com/aws/smithy-go v1.17.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 // indirect + github.com/aws/smithy-go v1.19.0 // indirect github.com/boltdb/bolt v1.3.1 // indirect github.com/bradenaw/juniper v0.13.1 // indirect github.com/buengese/sgzip v0.1.1 // indirect @@ -48,8 +48,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -cloud.google.com/go/storage v1.35.1 h1:B59ahL//eDfx2IIKFBeT5Atm9wnNmj3+8xG/W4WB//w= -cloud.google.com/go/storage v1.35.1/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= +cloud.google.com/go/storage v1.36.0 h1:P0mOkAcaJxhCTvAkMhxMfrTKiNcub4YmmPBtlhAyTr8= +cloud.google.com/go/storage v1.36.0/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= @@ -141,46 +141,46 @@ github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/aws/aws-sdk-go v1.48.4 h1:HS2L7ynVhkcRrQRro9CLJZ/xLRb4UOzDEfPzgevZwXM= github.com/aws/aws-sdk-go v1.48.4/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= -github.com/aws/aws-sdk-go-v2 v1.23.1 h1:qXaFsOOMA+HsZtX8WoCa+gJnbyW7qyFFBlPqvTSzbaI= -github.com/aws/aws-sdk-go-v2 v1.23.1/go.mod h1:i1XDttT4rnf6vxc9AuskLc6s7XBee8rlLilKlc03uAA= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 h1:ZY3108YtBNq96jNZTICHxN1gSBSbnvIdYwwqnvCV4Mc= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1/go.mod h1:t8PYl/6LzdAqsU4/9tz28V/kU+asFePvpOMkdul0gEQ= -github.com/aws/aws-sdk-go-v2/config v1.25.5 h1:UGKm9hpQS2hoK8CEJ1BzAW8NbUpvwDJJ4lyqXSzu8bk= -github.com/aws/aws-sdk-go-v2/config v1.25.5/go.mod h1:Bf4gDvy4ZcFIK0rqDu1wp9wrubNba2DojiPB2rt6nvI= -github.com/aws/aws-sdk-go-v2/credentials v1.16.4 h1:i7UQYYDSJrtc30RSwJwfBKwLFNnBTiICqAJ0pPdum8E= -github.com/aws/aws-sdk-go-v2/credentials v1.16.4/go.mod h1:Kdh/okh+//vQ/AjEt81CjvkTo64+/zIE4OewP7RpfXk= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.5 h1:KehRNiVzIfAcj6gw98zotVbb/K67taJE0fkfgM6vzqU= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.5/go.mod h1:VhnExhw6uXy9QzetvpXDolo1/hjhx4u9qukBGkuUwjs= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.4 h1:LAm3Ycm9HJfbSCd5I+wqC2S9Ej7FPrgr5CQoOljJZcE= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.4/go.mod h1:xEhvbJcyUf/31yfGSQBe01fukXwXJ0gxDp7rLfymWE0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.4 h1:4GV0kKZzUxiWxSVpn/9gwR0g21NF1Jsyduzo9rHgC/Q= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.4/go.mod h1:dYvTNAggxDZy6y1AF7YDwXsPuHFy/VNEpEI/2dWK9IU= -github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 h1:uR9lXYjdPX0xY+NhvaJ4dD8rpSRz5VY81ccIIoNG+lw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.4 h1:40Q4X5ebZruRtknEZH/bg91sT5pR853F7/1X9QRbI54= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.4/go.mod h1:u77N7eEECzUv7F0xl2gcfK/vzc8wcjWobpy+DcrLJ5E= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1 h1:rpkF4n0CyFcrJUG/rNNohoTmhtWlFTRI4BsZOh9PvLs= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1/go.mod h1:l9ymW25HOqymeU2m1gbUQ3rUIsTwKs8gYHXkqDQUhiI= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.4 h1:6DRKQc+9cChgzL5gplRGusI5dBGeiEod4m/pmGbcX48= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.4/go.mod h1:s8ORvrW4g4v7IvYKIAoBg17w3GQ+XuwXDXYrQ5SkzU0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.4 h1:rdovz3rEu0vZKbzoMYPTehp0E8veoE9AyfzqCr5Eeao= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.4/go.mod h1:aYCGNjyUCUelhofxlZyj63srdxWUSsBSGg5l6MCuXuE= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.4 h1:o3DcfCxGDIT20pTbVKVhp3vWXOj/VvgazNJvumWeYW0= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.4/go.mod h1:Uy0KVOxuTK2ne+/PKQ+VvEeWmjMMksE17k/2RK/r5oM= -github.com/aws/aws-sdk-go-v2/service/s3 v1.44.0 h1:FJTWR2nP1ddLIbk4n7Glw8wGbeWGHaViUwADPzE/EBo= -github.com/aws/aws-sdk-go-v2/service/s3 v1.44.0/go.mod h1:dqJ5JBL0clzgHriH35Amx3LRFY6wNIPUX7QO/BerSBo= +github.com/aws/aws-sdk-go-v2 v1.24.0 h1:890+mqQ+hTpNuw0gGP6/4akolQkSToDJgHfQE7AwGuk= +github.com/aws/aws-sdk-go-v2 v1.24.0/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo= +github.com/aws/aws-sdk-go-v2/config v1.26.1 h1:z6DqMxclFGL3Zfo+4Q0rLnAZ6yVkzCRxhRMsiRQnD1o= +github.com/aws/aws-sdk-go-v2/config v1.26.1/go.mod h1:ZB+CuKHRbb5v5F0oJtGdhFTelmrxd4iWO1lf0rQwSAg= +github.com/aws/aws-sdk-go-v2/credentials v1.16.12 h1:v/WgB8NxprNvr5inKIiVVrXPuuTegM+K8nncFkr1usU= +github.com/aws/aws-sdk-go-v2/credentials v1.16.12/go.mod h1:X21k0FjEJe+/pauud82HYiQbEr9jRKY3kXEIQ4hXeTQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 h1:w98BT5w+ao1/r5sUuiH6JkVzjowOKeOJRHERyy1vh58= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10/go.mod h1:K2WGI7vUvkIv1HoNbfBA1bvIZ+9kL3YVmWxeKuLQsiw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 h1:v+HbZaCGmOwnTTVS86Fleq0vPzOd7tnJGbFhP0stNLs= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9/go.mod h1:Xjqy+Nyj7VDLBtCMkQYOw1QYfAEZCVLrfI0ezve8wd4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 h1:N94sVhRACtXyVcjXxrwK1SKFIJrA9pOJ5yu2eSHnmls= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9/go.mod h1:hqamLz7g1/4EJP+GH5NBhcUMLjW+gKLQabgyz6/7WAU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 h1:ugD6qzjYtB7zM5PN/ZIeaAIyefPaD82G8+SJopgvUpw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9/go.mod h1:YD0aYBWCrPENpHolhKw2XDlTIWae2GKXT1T4o6N6hiM= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 h1:/90OR2XbSYfXucBMJ4U14wrjlfleq/0SB6dZDPncgmo= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9/go.mod h1:dN/Of9/fNZet7UrQQ6kTDo/VSwKPIq94vjlU16bRARc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 h1:Nf2sHxjMJR8CSImIVCONRi4g0Su3J+TSTbS7G0pUeMU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9/go.mod h1:idky4TER38YIjr2cADF1/ugFMKvZV7p//pVeV5LZbF0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 h1:iEAeF6YC3l4FzlJPP9H3Ko1TXpdjdqWffxXjp8SY6uk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9/go.mod h1:kjsXoK23q9Z/tLBrckZLLyvjhZoS+AGrzqzUfEClvMM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5 h1:Keso8lIOS+IzI2MkPZyK6G0LYcK3My2LQ+T5bxghEAY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5/go.mod h1:vADO6Jn+Rq4nDtfwNjhgR84qkZwiC6FqCaXdw/kYwjA= github.com/aws/aws-sdk-go-v2/service/sns v1.25.3 h1:6/Esm0BnUNrx+yy8AaslbaeJa8V40tTJ9N+tOihYWVo= github.com/aws/aws-sdk-go-v2/service/sns v1.25.3/go.mod h1:GkPiLToDWySwNSsR4AVam/Sv8UAZuMlGe9dozvyRCPE= github.com/aws/aws-sdk-go-v2/service/sqs v1.28.2 h1:MVg4eLi9uM1+YHYSfcCg1CR3mqtL6UJ9SF3VrMxKmUE= github.com/aws/aws-sdk-go-v2/service/sqs v1.28.2/go.mod h1:7vHhhnzSGZcquR6+X7V+wDHdY8iOk5ge0z+FxoxkvJw= -github.com/aws/aws-sdk-go-v2/service/sso v1.17.3 h1:CdsSOGlFF3Pn+koXOIpTtvX7st0IuGsZ8kJqcWMlX54= -github.com/aws/aws-sdk-go-v2/service/sso v1.17.3/go.mod h1:oA6VjNsLll2eVuUoF2D+CMyORgNzPEW/3PyUdq6WQjI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.1 h1:cbRqFTVnJV+KRpwFl76GJdIZJKKCdTPnjUZ7uWh3pIU= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.1/go.mod h1:hHL974p5auvXlZPIjJTblXJpbkfK4klBczlsEaMCGVY= -github.com/aws/aws-sdk-go-v2/service/sts v1.25.4 h1:yEvZ4neOQ/KpUqyR+X0ycUTW/kVRNR4nDZ38wStHGAA= -github.com/aws/aws-sdk-go-v2/service/sts v1.25.4/go.mod h1:feTnm2Tk/pJxdX+eooEsxvlvTWBvDm6CasRZ+JOs2IY= -github.com/aws/smithy-go v1.17.0 h1:wWJD7LX6PBV6etBUwO0zElG0nWN9rUhp0WdYeHSHAaI= -github.com/aws/smithy-go v1.17.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 h1:ldSFWz9tEHAwHNmjx2Cvy1MjP5/L9kNoR0skc6wyOOM= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.5/go.mod h1:CaFfXLYL376jgbP7VKC96uFcU8Rlavak0UlAwk1Dlhc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 h1:2k9KmFawS63euAkY4/ixVNsYYwrwnd5fIvgEKkfZFNM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5/go.mod h1:W+nd4wWDVkSUIox9bacmkBP5NMFQeTJ/xqNabpzSR38= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 h1:5UYvv8JUvllZsRnfrcMQ+hJ9jNICmcgKPAO1CER25Wg= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.5/go.mod h1:XX5gh4CB7wAs4KhcF46G6C8a2i7eupU19dcAAE+EydU= +github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= +github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -636,8 +636,8 @@ github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/linxGnu/grocksdb v1.8.0 h1:H4L/LhP7GOMf1j17oQAElHgVlbEje2h14A8Tz9cM2BE= -github.com/linxGnu/grocksdb v1.8.0/go.mod h1:09CeBborffXhXdNpEcOeZrLKEnRtrZFEpFdPNI9Zjjg= +github.com/linxGnu/grocksdb v1.8.10 h1:6FAhBThErRfJaevGOZISYvkG7RD4gfzeq452X4r8pes= +github.com/linxGnu/grocksdb v1.8.10/go.mod h1:xZCIb5Muw+nhbDK4Y5UJuOrin5MceOuiXkVUR7vp4WY= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed h1:036IscGBfJsFIgJQzlui7nK1Ncm0tp2ktmPj8xO4N/0= github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= @@ -1056,8 +1056,8 @@ golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2Uz golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= -golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= diff --git a/k8s/charts/seaweedfs/Chart.yaml b/k8s/charts/seaweedfs/Chart.yaml index fe25e136e..33ec8344a 100644 --- a/k8s/charts/seaweedfs/Chart.yaml +++ b/k8s/charts/seaweedfs/Chart.yaml @@ -2,4 +2,4 @@ apiVersion: v1 description: SeaweedFS name: seaweedfs appVersion: "3.59" -version: 3.59.4 +version: 3.60.0 diff --git a/k8s/charts/seaweedfs/templates/_helpers.tpl b/k8s/charts/seaweedfs/templates/_helpers.tpl index dd118c86b..e9f35447d 100644 --- a/k8s/charts/seaweedfs/templates/_helpers.tpl +++ b/k8s/charts/seaweedfs/templates/_helpers.tpl @@ -163,3 +163,22 @@ imagePullSecrets: {{- end }} {{- end }} {{- end -}} + +{{/* +Renders a value that contains template perhaps with scope if the scope is present. +Usage: +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }} +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }} +*/}} +{{- define "common.tplvalues.render" -}} +{{- $value := typeIs "string" .value | ternary .value (.value | toYaml) }} +{{- if contains "{{" (toJson .value) }} + {{- if .scope }} + {{- tpl (cat "{{- with $.RelativeScope -}}" $value "{{- end }}") (merge (dict "RelativeScope" .scope) .context) }} + {{- else }} + {{- tpl $value .context }} + {{- end }} +{{- else }} + {{- $value }} +{{- end }} +{{- end -}} diff --git a/k8s/charts/seaweedfs/templates/filer-statefulset.yaml b/k8s/charts/seaweedfs/templates/filer-statefulset.yaml index a92d3cdd1..6ee847b91 100644 --- a/k8s/charts/seaweedfs/templates/filer-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/filer-statefulset.yaml @@ -258,6 +258,9 @@ spec: resources: {{ tpl .Values.filer.resources . | nindent 12 | trim }} {{- end }} + {{- if .Values.filer.sidecars }} + {{- include "common.tplvalues.render" (dict "value" .Values.filer.sidecars "context" $) | nindent 8 }} + {{- end }} volumes: {{- if eq .Values.filer.logs.type "hostPath" }} - name: seaweedfs-filer-log-volume diff --git a/k8s/charts/seaweedfs/templates/master-statefulset.yaml b/k8s/charts/seaweedfs/templates/master-statefulset.yaml index 19c30f07d..6e432450a 100644 --- a/k8s/charts/seaweedfs/templates/master-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/master-statefulset.yaml @@ -212,6 +212,9 @@ spec: resources: {{ tpl .Values.master.resources . | nindent 12 | trim }} {{- end }} + {{- if .Values.master.sidecars }} + {{- include "common.tplvalues.render" (dict "value" .Values.master.sidecars "context" $) | nindent 8 }} + {{- end }} volumes: {{- if eq .Values.master.logs.type "hostPath" }} - name: seaweedfs-master-log-volume diff --git a/k8s/charts/seaweedfs/templates/post-install-bucket-hook.yaml b/k8s/charts/seaweedfs/templates/post-install-bucket-hook.yaml index b8c95fcca..d9f7e6bcf 100644 --- a/k8s/charts/seaweedfs/templates/post-install-bucket-hook.yaml +++ b/k8s/charts/seaweedfs/templates/post-install-bucket-hook.yaml @@ -80,9 +80,6 @@ spec: {{- end }} - containerPort: {{ .Values.master.grpcPort }} #name: swfs-master-grpc - {{- if .Values.master.readinessProbe.enabled }} - {{- $hostpath_exists := include "master.hostpath_exists" . -}} - {{- $existing_claims := include "master.existing_claims" . -}} {{- if .Values.filer.s3.enableAuth }} volumes: - name: config-users @@ -93,8 +90,7 @@ spec: {{- else }} secretName: seaweedfs-s3-secret {{- end }} - {{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} + {{- end }}{{/** if .Values.filer.s3.enableAuth **/}} +{{- end }}{{/** if .Values.master.enabled **/}} +{{- end }}{{/** if .Values.filer.s3.enabled **/}} +{{- end }}{{/** if .Values.filer.s3.createBuckets **/}} diff --git a/k8s/charts/seaweedfs/templates/s3-deployment.yaml b/k8s/charts/seaweedfs/templates/s3-deployment.yaml index c08873fa5..45b0d394b 100644 --- a/k8s/charts/seaweedfs/templates/s3-deployment.yaml +++ b/k8s/charts/seaweedfs/templates/s3-deployment.yaml @@ -183,6 +183,9 @@ spec: resources: {{ tpl .Values.s3.resources . | nindent 12 | trim }} {{- end }} + {{- if .Values.s3.sidecars }} + {{- include "common.tplvalues.render" (dict "value" .Values.s3.sidecars "context" $) | nindent 8 }} + {{- end }} volumes: {{- if .Values.s3.enableAuth }} - name: config-users diff --git a/k8s/charts/seaweedfs/templates/volume-statefulset.yaml b/k8s/charts/seaweedfs/templates/volume-statefulset.yaml index 1b8c9267e..5375a4b7c 100644 --- a/k8s/charts/seaweedfs/templates/volume-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/volume-statefulset.yaml @@ -218,6 +218,9 @@ spec: resources: {{ tpl .Values.volume.resources . | nindent 12 | trim }} {{- end }} + {{- if .Values.volume.sidecars }} + {{- include "common.tplvalues.render" (dict "value" .Values.volume.sidecars "context" $) | nindent 8 }} + {{- end }} volumes: {{- if eq .Values.volume.data.type "hostPath" }} - name: data diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index 7a2b04d64..a0138d15c 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -98,7 +98,18 @@ master: size: "" storageClass: "" hostPathPrefix: /storage - + + ## @param master.sidecars Add additional sidecar containers to the master pod(s) + ## e.g: + ## sidecars: + ## - name: your-image-name + ## image: your-image + ## imagePullPolicy: Always + ## ports: + ## - name: portname + ## containerPort: 1234 + ## + sidecars: [] initContainers: "" extraVolumes: "" @@ -286,7 +297,18 @@ volume: # Adjust jpg orientation when uploading. imagesFixOrientation: false - + + ## @param volume.sidecars Add additional sidecar containers to the volume pod(s) + ## e.g: + ## sidecars: + ## - name: your-image-name + ## image: your-image + ## imagePullPolicy: Always + ## ports: + ## - name: portname + ## containerPort: 1234 + ## + sidecars: [] initContainers: "" extraVolumes: "" @@ -429,7 +451,18 @@ filer: size: "" storageClass: "" hostPathPrefix: /storage - + + ## @param filer.sidecars Add additional sidecar containers to the filer pod(s) + ## e.g: + ## sidecars: + ## - name: your-image-name + ## image: your-image + ## imagePullPolicy: Always + ## ports: + ## - name: portname + ## containerPort: 1234 + ## + sidecars: [] initContainers: "" extraVolumes: "" @@ -611,7 +644,18 @@ s3: # Suffix of the host name, {bucket}.{domainName} domainName: "" - + + ## @param s3.sidecars Add additional sidecar containers to the s3 pod(s) + ## e.g: + ## sidecars: + ## - name: your-image-name + ## image: your-image + ## imagePullPolicy: Always + ## ports: + ## - name: portname + ## containerPort: 1234 + ## + sidecars: [] initContainers: "" extraVolumes: "" diff --git a/weed/command/filer.go b/weed/command/filer.go index fe0beb5b8..50fc4492b 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -61,6 +61,7 @@ type FilerOptions struct { showUIDirectoryDelete *bool downloadMaxMBps *int diskType *string + allowedOrigins *string } func init() { @@ -91,6 +92,7 @@ func init() { f.showUIDirectoryDelete = cmdFiler.Flag.Bool("ui.deleteDir", true, "enable filer UI show delete directory button") f.downloadMaxMBps = cmdFiler.Flag.Int("downloadMaxMBps", 0, "download max speed for each download request, in MB per second") f.diskType = cmdFiler.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag") + f.allowedOrigins = cmdFiler.Flag.String("allowedOrigins", "*", "comma separated list of allowed origins") // start s3 on filer filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway") @@ -229,6 +231,9 @@ func (fo *FilerOptions) startFiler() { if *fo.bindIp == "" { *fo.bindIp = *fo.ip } + if *fo.allowedOrigins == "" { + *fo.allowedOrigins = "*" + } defaultLevelDbDirectory := util.ResolvePath(*fo.defaultLevelDbDirectory + "/filerldb2") @@ -253,6 +258,7 @@ func (fo *FilerOptions) startFiler() { ShowUIDirectoryDelete: *fo.showUIDirectoryDelete, DownloadMaxBytesPs: int64(*fo.downloadMaxMBps) * 1024 * 1024, DiskType: *fo.diskType, + AllowedOrigins: strings.Split(*fo.allowedOrigins, ","), }) if nfs_err != nil { glog.Fatalf("Filer startup error: %v", nfs_err) diff --git a/weed/command/filer_remote_gateway_buckets.go b/weed/command/filer_remote_gateway_buckets.go index 9694a1c9c..912607847 100644 --- a/weed/command/filer_remote_gateway_buckets.go +++ b/weed/command/filer_remote_gateway_buckets.go @@ -30,10 +30,20 @@ func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSo return err } - processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error { - lastTime := time.Unix(0, lastTsNs) - glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3)) - return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, lastTsNs) + processor := NewMetadataProcessor(eachEntryFunc, 128) + + var lastLogTsNs = time.Now().UnixNano() + processEventFnWithOffset := pb.AddOffsetFunc(func(resp *filer_pb.SubscribeMetadataResponse) error { + processor.AddSyncJob(resp) + return nil + }, 3*time.Second, func(counter int64, lastTsNs int64) error { + if processor.processedTsWatermark == 0 { + return nil + } + now := time.Now().UnixNano() + glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, time.Unix(0, processor.processedTsWatermark), float64(counter)/(float64(now-lastLogTsNs)/1e9)) + lastLogTsNs = now + return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, processor.processedTsWatermark) }) lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, *option.timeAgo) diff --git a/weed/command/s3.go b/weed/command/s3.go index dc943b23d..b7bb2a546 100644 --- a/weed/command/s3.go +++ b/weed/command/s3.go @@ -10,6 +10,7 @@ import ( "net/http" "os" "runtime" + "strings" "time" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" @@ -42,6 +43,7 @@ type S3Options struct { portGrpc *int config *string domainName *string + allowedOrigins *string tlsPrivateKey *string tlsCertificate *string tlsCACertificate *string @@ -64,6 +66,7 @@ func init() { s3StandaloneOptions.portHttps = cmdS3.Flag.Int("port.https", 0, "s3 server https listen port") s3StandaloneOptions.portGrpc = cmdS3.Flag.Int("port.grpc", 0, "s3 server grpc listen port") s3StandaloneOptions.domainName = cmdS3.Flag.String("domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}") + s3StandaloneOptions.allowedOrigins = cmdS3.Flag.String("allowedOrigins", "*", "comma separated list of allowed origins") s3StandaloneOptions.dataCenter = cmdS3.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center") s3StandaloneOptions.config = cmdS3.Flag.String("config", "", "path to the config file") s3StandaloneOptions.auditLogConfig = cmdS3.Flag.String("auditLogConfig", "", "path to the audit log config file") @@ -220,6 +223,7 @@ func (s3opt *S3Options) startS3Server() bool { Port: *s3opt.port, Config: *s3opt.config, DomainName: *s3opt.domainName, + AllowedOrigins: strings.Split(*s3opt.allowedOrigins, ","), BucketsPath: filerBucketsPath, GrpcDialOption: grpcDialOption, AllowEmptyFolder: *s3opt.allowEmptyFolder, diff --git a/weed/command/scaffold/filer.toml b/weed/command/scaffold/filer.toml index 55876bea0..231e7510a 100644 --- a/weed/command/scaffold/filer.toml +++ b/weed/command/scaffold/filer.toml @@ -51,6 +51,9 @@ dbFile = "./filer.db" # sqlite db file # ) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; enabled = false +# dsn will take priority over "hostname, port, username, password, database". +# [username[:password]@][protocol[(address)]]/dbname[?param1=value1&...¶mN=valueN] +dsn = "root@tcp(localhost:3306)/seaweedfs?collation=utf8mb4_bin" hostname = "localhost" port = 3306 username = "root" diff --git a/weed/command/scaffold/security.toml b/weed/command/scaffold/security.toml index e5452cdff..9626ee58c 100644 --- a/weed/command/scaffold/security.toml +++ b/weed/command/scaffold/security.toml @@ -4,6 +4,11 @@ # /etc/seaweedfs/security.toml # this file is read by master, volume server, and filer +# comma separated origins allowed to make requests to the filer and s3 gateway. +# enter in this format: https://domain.com, or http://localhost:port +[cors.allowed_origins] +values = "*" + # this jwt signing key is read by master and volume server, and it is used for write operations: # - the Master server generates the JWT, which can be used to write a certain file on a volume server # - the Volume server validates the JWT on writing diff --git a/weed/command/server.go b/weed/command/server.go index 67e37426e..9631f6bfd 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -106,6 +106,7 @@ func init() { filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port") filerOptions.portGrpc = cmdServer.Flag.Int("filer.port.grpc", 0, "filer server grpc listen port") filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port") + filerOptions.allowedOrigins = cmdServer.Flag.String("filer.allowedOrigins", "*", "comma separated list of allowed origins") filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.") filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing") filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 4, "split files larger than the limit") @@ -142,6 +143,7 @@ func init() { s3Options.portHttps = cmdServer.Flag.Int("s3.port.https", 0, "s3 server https listen port") s3Options.portGrpc = cmdServer.Flag.Int("s3.port.grpc", 0, "s3 server grpc listen port") s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}") + s3Options.allowedOrigins = cmdServer.Flag.String("s3.allowedOrigins", "*", "comma separated list of allowed origins") s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file") s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file") s3Options.tlsCACertificate = cmdServer.Flag.String("s3.cacert.file", "", "path to the TLS CA certificate file") diff --git a/weed/filer/filer.go b/weed/filer/filer.go index 239263ca8..1c6b3c338 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -3,12 +3,13 @@ package filer import ( "context" "fmt" - "github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager" "os" "sort" "strings" "time" + "github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager" + "github.com/seaweedfs/seaweedfs/weed/cluster" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" diff --git a/weed/filer/mysql/mysql_store.go b/weed/filer/mysql/mysql_store.go index 14566d49b..f1a246575 100644 --- a/weed/filer/mysql/mysql_store.go +++ b/weed/filer/mysql/mysql_store.go @@ -3,6 +3,8 @@ package mysql import ( "database/sql" "fmt" + "github.com/go-sql-driver/mysql" + "strings" "time" "github.com/seaweedfs/seaweedfs/weed/filer" @@ -30,6 +32,7 @@ func (store *MysqlStore) GetName() string { func (store *MysqlStore) Initialize(configuration util.Configuration, prefix string) (err error) { return store.initialize( + configuration.GetString(prefix+"dsn"), configuration.GetString(prefix+"upsertQuery"), configuration.GetBool(prefix+"enableUpsert"), configuration.GetString(prefix+"username"), @@ -44,7 +47,7 @@ func (store *MysqlStore) Initialize(configuration util.Configuration, prefix str ) } -func (store *MysqlStore) initialize(upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database string, maxIdle, maxOpen, +func (store *MysqlStore) initialize(dsn string, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database string, maxIdle, maxOpen, maxLifetimeSeconds int, interpolateParams bool) (err error) { store.SupportBucketTable = false @@ -57,19 +60,23 @@ func (store *MysqlStore) initialize(upsertQuery string, enableUpsert bool, user, UpsertQueryTemplate: upsertQuery, } - sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database) - adaptedSqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, "<ADAPTED>", hostname, port, database) - if interpolateParams { - sqlUrl += "&interpolateParams=true" - adaptedSqlUrl += "&interpolateParams=true" + if dsn == "" { + dsn = fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database) + if interpolateParams { + dsn += "&interpolateParams=true" + } + } + cfg, err := mysql.ParseDSN(dsn) + if err != nil { + return fmt.Errorf("can not parse DSN error:%v", err) } var dbErr error - store.DB, dbErr = sql.Open("mysql", sqlUrl) + store.DB, dbErr = sql.Open("mysql", dsn) if dbErr != nil { store.DB.Close() store.DB = nil - return fmt.Errorf("can not connect to %s error:%v", adaptedSqlUrl, err) + return fmt.Errorf("can not connect to %s error:%v", strings.ReplaceAll(dsn, cfg.Passwd, "<ADAPTED>"), err) } store.DB.SetMaxIdleConns(maxIdle) @@ -77,7 +84,7 @@ func (store *MysqlStore) initialize(upsertQuery string, enableUpsert bool, user, store.DB.SetConnMaxLifetime(time.Duration(maxLifetimeSeconds) * time.Second) if err = store.DB.Ping(); err != nil { - return fmt.Errorf("connect to %s error:%v", sqlUrl, err) + return fmt.Errorf("connect to %s error:%v", strings.ReplaceAll(dsn, cfg.Passwd, "<ADAPTED>"), err) } return nil diff --git a/weed/s3api/s3api_handlers.go b/weed/s3api/s3api_handlers.go index 81d7017dc..c146a8b15 100644 --- a/weed/s3api/s3api_handlers.go +++ b/weed/s3api/s3api_handlers.go @@ -40,6 +40,10 @@ func writeSuccessResponseEmpty(w http.ResponseWriter, r *http.Request) { s3err.WriteEmptyResponse(w, r, http.StatusOK) } +func writeFailureResponse(w http.ResponseWriter, r *http.Request, errCode s3err.ErrorCode) { + s3err.WriteErrorResponse(w, r, errCode) +} + func validateContentMd5(h http.Header) ([]byte, error) { md5B64, ok := h["Content-Md5"] if ok { diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index e90c334aa..37ae54f1b 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -3,15 +3,16 @@ package s3api import ( "context" "fmt" - "github.com/seaweedfs/seaweedfs/weed/filer" - "github.com/seaweedfs/seaweedfs/weed/glog" - "github.com/seaweedfs/seaweedfs/weed/pb/s3_pb" - "github.com/seaweedfs/seaweedfs/weed/util/grace" "net" "net/http" "strings" "time" + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/pb/s3_pb" + "github.com/seaweedfs/seaweedfs/weed/util/grace" + "github.com/gorilla/mux" "github.com/seaweedfs/seaweedfs/weed/pb" . "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" @@ -26,6 +27,7 @@ type S3ApiServerOption struct { Port int Config string DomainName string + AllowedOrigins []string BucketsPath string GrpcDialOption grpc.DialOption AllowEmptyFolder bool @@ -56,6 +58,14 @@ func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer v.SetDefault("jwt.filer_signing.read.expires_after_seconds", 60) readExpiresAfterSec := v.GetInt("jwt.filer_signing.read.expires_after_seconds") + v.SetDefault("cors.allowed_origins.values", "*") + + if (option.AllowedOrigins == nil) || (len(option.AllowedOrigins) == 0) { + allowedOrigins := v.GetString("cors.allowed_origins.values") + domains := strings.Split(allowedOrigins, ",") + option.AllowedOrigins = domains + } + s3ApiServer = &S3ApiServer{ option: option, iam: NewIdentityAccessManagement(option), @@ -103,7 +113,25 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { apiRouter.Methods("OPTIONS").HandlerFunc( func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Access-Control-Allow-Origin", "*") + origin := r.Header.Get("Origin") + if origin != "" { + if s3a.option.AllowedOrigins == nil || len(s3a.option.AllowedOrigins) == 0 || s3a.option.AllowedOrigins[0] == "*" { + origin = "*" + } else { + originFound := false + for _, allowedOrigin := range s3a.option.AllowedOrigins { + if origin == allowedOrigin { + originFound = true + } + } + if !originFound { + writeFailureResponse(w, r, http.StatusForbidden) + return + } + } + } + + w.Header().Set("Access-Control-Allow-Origin", origin) w.Header().Set("Access-Control-Expose-Headers", "*") w.Header().Set("Access-Control-Allow-Methods", "*") w.Header().Set("Access-Control-Allow-Headers", "*") diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 1b50d47c7..20b5151cd 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "os" + "strings" "sync" "time" @@ -70,6 +71,7 @@ type FilerOption struct { ShowUIDirectoryDelete bool DownloadMaxBytesPs int64 DiskType string + AllowedOrigins []string } type FilerServer struct { @@ -107,6 +109,14 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) v.SetDefault("jwt.filer_signing.read.expires_after_seconds", 60) readExpiresAfterSec := v.GetInt("jwt.filer_signing.read.expires_after_seconds") + v.SetDefault("cors.allowed_origins.values", "*") + + if (option.AllowedOrigins == nil) || (len(option.AllowedOrigins) == 0) { + allowedOrigins := v.GetString("cors.allowed_origins.values") + domains := strings.Split(allowedOrigins, ",") + option.AllowedOrigins = domains + } + fs = &FilerServer{ option: option, grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.filer"), diff --git a/weed/server/filer_server_handlers.go b/weed/server/filer_server_handlers.go index 6bfae3dc1..d71b60d70 100644 --- a/weed/server/filer_server_handlers.go +++ b/weed/server/filer_server_handlers.go @@ -3,6 +3,7 @@ package weed_server import ( "errors" "net/http" + "os" "strings" "sync/atomic" "time" @@ -17,8 +18,24 @@ import ( func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { start := time.Now() - if r.Header.Get("Origin") != "" { - w.Header().Set("Access-Control-Allow-Origin", "*") + origin := r.Header.Get("Origin") + if origin != "" { + if fs.option.AllowedOrigins == nil || len(fs.option.AllowedOrigins) == 0 || fs.option.AllowedOrigins[0] == "*" { + origin = "*" + } else { + originFound := false + for _, allowedOrigin := range fs.option.AllowedOrigins { + if origin == allowedOrigin { + originFound = true + } + } + if !originFound { + writeJsonError(w, r, http.StatusForbidden, errors.New("origin not allowed")) + return + } + } + + w.Header().Set("Access-Control-Allow-Origin", origin) w.Header().Set("Access-Control-Expose-Headers", "*") w.Header().Set("Access-Control-Allow-Headers", "*") w.Header().Set("Access-Control-Allow-Credentials", "true") @@ -99,9 +116,27 @@ func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Reque start := time.Now() - if r.Header.Get("Origin") != "" { - w.Header().Set("Access-Control-Allow-Origin", "*") - w.Header().Set("Access-Control-Allow-Headers", "*") + os.Stdout.WriteString("Request: " + r.Method + " " + r.URL.String() + "\n") + + origin := r.Header.Get("Origin") + if origin != "" { + if fs.option.AllowedOrigins == nil || len(fs.option.AllowedOrigins) == 0 || fs.option.AllowedOrigins[0] == "*" { + origin = "*" + } else { + originFound := false + for _, allowedOrigin := range fs.option.AllowedOrigins { + if origin == allowedOrigin { + originFound = true + } + } + if !originFound { + writeJsonError(w, r, http.StatusForbidden, errors.New("origin not allowed")) + return + } + } + + w.Header().Set("Access-Control-Allow-Origin", origin) + w.Header().Set("Access-Control-Allow-Headers", "OPTIONS, GET, HEAD") w.Header().Set("Access-Control-Allow-Credentials", "true") } @@ -137,7 +172,6 @@ func OptionsHandler(w http.ResponseWriter, r *http.Request, isReadOnly bool) { w.Header().Set("Access-Control-Allow-Methods", "PUT, POST, GET, DELETE, OPTIONS") w.Header().Set("Access-Control-Expose-Headers", "*") } - w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Access-Control-Allow-Headers", "*") w.Header().Set("Access-Control-Allow-Credentials", "true") } |