diff options
| author | chrislu <chris.lu@gmail.com> | 2024-08-10 10:01:57 -0700 |
|---|---|---|
| committer | chrislu <chris.lu@gmail.com> | 2024-08-10 10:01:57 -0700 |
| commit | 7438648d1cfacd5ca570dd029d1bdb5fd271bd70 (patch) | |
| tree | cf12b49473be0373cb03d83470ddc75708454171 | |
| parent | 49893267e978cc3fda00dc991e00099742fb5a9d (diff) | |
| parent | 63c707f9c1b4dc469ec39c446563c324ce4ccb6f (diff) | |
| download | seaweedfs-7438648d1cfacd5ca570dd029d1bdb5fd271bd70.tar.xz seaweedfs-7438648d1cfacd5ca570dd029d1bdb5fd271bd70.zip | |
Merge branch 'master' into mq
243 files changed, 5780 insertions, 2534 deletions
diff --git a/.github/workflows/binaries_dev.yml b/.github/workflows/binaries_dev.yml index 215eee2ba..84845817c 100644 --- a/.github/workflows/binaries_dev.yml +++ b/.github/workflows/binaries_dev.yml @@ -44,7 +44,7 @@ jobs: run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV} - name: Go Release Binaries Large Disk - uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22 + uses: wangyoucao577/go-release-action@6ac7dba1f9e61850053324549cb6bc88e4b473d2 # v1.22 with: github_token: ${{ secrets.GITHUB_TOKEN }} goos: ${{ matrix.goos }} @@ -60,7 +60,7 @@ jobs: asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" - name: Go Release Binaries Normal Volume Size - uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22 + uses: wangyoucao577/go-release-action@6ac7dba1f9e61850053324549cb6bc88e4b473d2 # v1.22 with: github_token: ${{ secrets.GITHUB_TOKEN }} goos: ${{ matrix.goos }} @@ -93,7 +93,7 @@ jobs: run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV} - name: Go Release Binaries Large Disk - uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22 + uses: wangyoucao577/go-release-action@6ac7dba1f9e61850053324549cb6bc88e4b473d2 # v1.22 with: github_token: ${{ secrets.GITHUB_TOKEN }} goos: ${{ matrix.goos }} @@ -109,7 +109,7 @@ jobs: asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" - name: Go Release Binaries Normal Volume Size - uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22 + uses: wangyoucao577/go-release-action@6ac7dba1f9e61850053324549cb6bc88e4b473d2 # v1.22 with: github_token: ${{ secrets.GITHUB_TOKEN }} goos: ${{ matrix.goos }} diff --git a/.github/workflows/binaries_release0.yml b/.github/workflows/binaries_release0.yml index dfb7e4bb8..c59ad26dd 100644 --- a/.github/workflows/binaries_release0.yml +++ b/.github/workflows/binaries_release0.yml @@ -30,7 +30,7 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2 - name: Go Release Binaries Normal Volume Size - uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22 + uses: wangyoucao577/go-release-action@6ac7dba1f9e61850053324549cb6bc88e4b473d2 # v1.22 with: github_token: ${{ secrets.GITHUB_TOKEN }} goos: ${{ matrix.goos }} @@ -44,7 +44,7 @@ jobs: binary_name: weed asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}" - name: Go Release Large Disk Binaries - uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22 + uses: wangyoucao577/go-release-action@6ac7dba1f9e61850053324549cb6bc88e4b473d2 # v1.22 with: github_token: ${{ secrets.GITHUB_TOKEN }} goos: ${{ matrix.goos }} diff --git a/.github/workflows/binaries_release1.yml b/.github/workflows/binaries_release1.yml index 4a3c85c68..10120c2dd 100644 --- a/.github/workflows/binaries_release1.yml +++ b/.github/workflows/binaries_release1.yml @@ -30,7 +30,7 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2 - name: Go Release Binaries Normal Volume Size - uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22 + uses: wangyoucao577/go-release-action@6ac7dba1f9e61850053324549cb6bc88e4b473d2 # v1.22 with: github_token: ${{ secrets.GITHUB_TOKEN }} goos: ${{ matrix.goos }} @@ -44,7 +44,7 @@ jobs: binary_name: weed asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}" - name: Go Release Large Disk Binaries - uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22 + uses: wangyoucao577/go-release-action@6ac7dba1f9e61850053324549cb6bc88e4b473d2 # v1.22 with: github_token: ${{ secrets.GITHUB_TOKEN }} goos: ${{ matrix.goos }} diff --git a/.github/workflows/binaries_release2.yml b/.github/workflows/binaries_release2.yml index 0061de42b..4efa6509a 100644 --- a/.github/workflows/binaries_release2.yml +++ b/.github/workflows/binaries_release2.yml @@ -30,7 +30,7 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2 - name: Go Release Binaries Normal Volume Size - uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22 + uses: wangyoucao577/go-release-action@6ac7dba1f9e61850053324549cb6bc88e4b473d2 # v1.22 with: github_token: ${{ secrets.GITHUB_TOKEN }} goos: ${{ matrix.goos }} @@ -44,7 +44,7 @@ jobs: binary_name: weed asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}" - name: Go Release Large Disk Binaries - uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22 + uses: wangyoucao577/go-release-action@6ac7dba1f9e61850053324549cb6bc88e4b473d2 # v1.22 with: github_token: ${{ secrets.GITHUB_TOKEN }} goos: ${{ matrix.goos }} diff --git a/.github/workflows/binaries_release3.yml b/.github/workflows/binaries_release3.yml index 727b11019..fa8405067 100644 --- a/.github/workflows/binaries_release3.yml +++ b/.github/workflows/binaries_release3.yml @@ -30,7 +30,7 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2 - name: Go Release Binaries Normal Volume Size - uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22 + uses: wangyoucao577/go-release-action@6ac7dba1f9e61850053324549cb6bc88e4b473d2 # v1.22 with: github_token: ${{ secrets.GITHUB_TOKEN }} goos: ${{ matrix.goos }} @@ -44,7 +44,7 @@ jobs: binary_name: weed asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}" - name: Go Release Large Disk Binaries - uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22 + uses: wangyoucao577/go-release-action@6ac7dba1f9e61850053324549cb6bc88e4b473d2 # v1.22 with: github_token: ${{ secrets.GITHUB_TOKEN }} goos: ${{ matrix.goos }} diff --git a/.github/workflows/binaries_release4.yml b/.github/workflows/binaries_release4.yml index 3fcd2ee47..cf760a816 100644 --- a/.github/workflows/binaries_release4.yml +++ b/.github/workflows/binaries_release4.yml @@ -30,7 +30,7 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2 - name: Go Release Binaries Normal Volume Size - uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22 + uses: wangyoucao577/go-release-action@6ac7dba1f9e61850053324549cb6bc88e4b473d2 # v1.22 with: github_token: ${{ secrets.GITHUB_TOKEN }} goos: ${{ matrix.goos }} @@ -45,7 +45,7 @@ jobs: binary_name: weed asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full" - name: Go Release Large Disk Binaries - uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22 + uses: wangyoucao577/go-release-action@6ac7dba1f9e61850053324549cb6bc88e4b473d2 # v1.22 with: github_token: ${{ secrets.GITHUB_TOKEN }} goos: ${{ matrix.goos }} diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml index a075022dd..eb8524493 100644 --- a/.github/workflows/container_dev.yml +++ b/.github/workflows/container_dev.yml @@ -33,30 +33,30 @@ jobs: org.opencontainers.image.vendor=Chris Lu - name: Set up QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 + uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1 + uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v1 with: buildkitd-flags: "--debug" - name: Login to Docker Hub if: github.event_name != 'pull_request' - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1 + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Login to GHCR if: github.event_name != 'pull_request' - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1 + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1 with: registry: ghcr.io username: ${{ secrets.GHCR_USERNAME }} password: ${{ secrets.GHCR_TOKEN }} - name: Build - uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v2 + uses: docker/build-push-action@5176d81f87c23d6fc96624dfdbcd9f3830bbe445 # v2 with: context: ./docker push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml index ba14dac1f..425e26028 100644 --- a/.github/workflows/container_latest.yml +++ b/.github/workflows/container_latest.yml @@ -34,30 +34,30 @@ jobs: org.opencontainers.image.vendor=Chris Lu - name: Set up QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 + uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1 + uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v1 with: buildkitd-flags: "--debug" - name: Login to Docker Hub if: github.event_name != 'pull_request' - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1 + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Login to GHCR if: github.event_name != 'pull_request' - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1 + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1 with: registry: ghcr.io username: ${{ secrets.GHCR_USERNAME }} password: ${{ secrets.GHCR_TOKEN }} - name: Build - uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v2 + uses: docker/build-push-action@5176d81f87c23d6fc96624dfdbcd9f3830bbe445 # v2 with: context: ./docker push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/container_release1.yml b/.github/workflows/container_release1.yml index 1374c07a1..1fc3e9145 100644 --- a/.github/workflows/container_release1.yml +++ b/.github/workflows/container_release1.yml @@ -34,20 +34,20 @@ jobs: org.opencontainers.image.vendor=Chris Lu - name: Set up QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 + uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1 + uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1 + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build - uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v2 + uses: docker/build-push-action@5176d81f87c23d6fc96624dfdbcd9f3830bbe445 # v2 with: context: ./docker push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/container_release2.yml b/.github/workflows/container_release2.yml index bad1adf8f..69eb1e356 100644 --- a/.github/workflows/container_release2.yml +++ b/.github/workflows/container_release2.yml @@ -35,20 +35,20 @@ jobs: org.opencontainers.image.vendor=Chris Lu - name: Set up QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 + uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1 + uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1 + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build - uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v2 + uses: docker/build-push-action@5176d81f87c23d6fc96624dfdbcd9f3830bbe445 # v2 with: context: ./docker push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/container_release3.yml b/.github/workflows/container_release3.yml index af6bb78f0..356321fb0 100644 --- a/.github/workflows/container_release3.yml +++ b/.github/workflows/container_release3.yml @@ -35,20 +35,20 @@ jobs: org.opencontainers.image.vendor=Chris Lu - name: Set up QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 + uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1 + uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1 + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build - uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v2 + uses: docker/build-push-action@5176d81f87c23d6fc96624dfdbcd9f3830bbe445 # v2 with: context: ./docker push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/container_release4.yml b/.github/workflows/container_release4.yml index 232d44c1f..2b4682adb 100644 --- a/.github/workflows/container_release4.yml +++ b/.github/workflows/container_release4.yml @@ -34,20 +34,20 @@ jobs: org.opencontainers.image.vendor=Chris Lu - name: Set up QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 + uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1 + uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1 + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build - uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v2 + uses: docker/build-push-action@5176d81f87c23d6fc96624dfdbcd9f3830bbe445 # v2 with: context: ./docker push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/container_release5.yml b/.github/workflows/container_release5.yml index bea6f346b..fc12fe6a0 100644 --- a/.github/workflows/container_release5.yml +++ b/.github/workflows/container_release5.yml @@ -34,20 +34,20 @@ jobs: org.opencontainers.image.vendor=Chris Lu - name: Set up QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 + uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1 + uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v1 + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build - uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v2 + uses: docker/build-push-action@5176d81f87c23d6fc96624dfdbcd9f3830bbe445 # v2 with: context: ./docker push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml index 866f2d888..f7f762fc1 100644 --- a/.github/workflows/s3tests.yml +++ b/.github/workflows/s3tests.yml @@ -34,7 +34,7 @@ jobs: go-version-file: 'go.mod' id: go - - name: Run Ceph S3 tests + - name: Run Ceph S3 tests with KV store timeout-minutes: 15 env: S3TEST_CONF: /__w/seaweedfs/seaweedfs/docker/compose/s3tests.conf @@ -43,17 +43,21 @@ jobs: cd /__w/seaweedfs/seaweedfs/weed go install -buildvcs=false set -x - nohup weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \ + weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \ -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \ -volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \ -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json & + pid=$! sleep 10 cd /s3-tests + sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests_boto3/functional/test_s3.py tox -- \ s3tests_boto3/functional/test_s3.py::test_bucket_list_empty \ s3tests_boto3/functional/test_s3.py::test_bucket_list_distinct \ s3tests_boto3/functional/test_s3.py::test_bucket_list_many \ s3tests_boto3/functional/test_s3.py::test_bucket_listv2_many \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_basic \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_encoding_basic \ s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix \ s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_ends_with_delimiter \ s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_alt \ @@ -94,6 +98,7 @@ jobs: s3tests_boto3/functional/test_s3.py::test_bucket_listv2_maxkeys_zero \ s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_none \ s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_empty \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_continuationtoken_empty \ s3tests_boto3/functional/test_s3.py::test_bucket_listv2_continuationtoken \ s3tests_boto3/functional/test_s3.py::test_bucket_listv2_both_continuationtoken_startafter \ s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_unreadable \ @@ -198,3 +203,84 @@ jobs: s3tests_boto3/functional/test_s3.py::test_lifecycle_set \ s3tests_boto3/functional/test_s3.py::test_lifecycle_get \ s3tests_boto3/functional/test_s3.py::test_lifecycle_set_filter + kill -9 $pid || true + + - name: Run Ceph S3 tests with SQL store + timeout-minutes: 15 + env: + S3TEST_CONF: /__w/seaweedfs/seaweedfs/docker/compose/s3tests.conf + shell: bash + run: | + cd /__w/seaweedfs/seaweedfs/weed + go install -tags "sqlite" -buildvcs=false + export WEED_LEVELDB2_ENABLED="false" WEED_SQLITE_ENABLED="true" WEED_SQLITE_DBFILE="./filer.db" + set -x + weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \ + -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \ + -volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \ + -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json & + pid=$! + sleep 10 + cd /s3-tests + sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests_boto3/functional/test_s3.py + tox -- \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_empty \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_distinct \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_many \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_many \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_basic \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_encoding_basic \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_ends_with_delimiter \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_alt \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_underscore \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_percentage \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_whitespace \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_dot \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_unreadable \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_empty \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_none \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_basic \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_basic \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_alt \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_alt \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_delimiter_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_delimiter_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_delimiter_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_fetchowner_notempty \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_basic \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_basic \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_alt \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_alt \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_empty \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_empty \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_none \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_none \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_not_exist \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_unreadable \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_unreadable \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_maxkeys_one \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_maxkeys_one \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_maxkeys_zero \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_maxkeys_zero \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_none \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_empty \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_continuationtoken_empty \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_continuationtoken \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_both_continuationtoken_startafter \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_unreadable \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_unreadable \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_not_in_list \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_not_in_list \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_after_list \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_after_list \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_objects_anonymous_fail \ + s3tests_boto3/functional/test_s3.py::test_bucket_listv2_objects_anonymous_fail \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_long_name \ + s3tests_boto3/functional/test_s3.py::test_bucket_list_special_prefix + kill -9 $pid || true @@ -34,6 +34,7 @@ Your support will be really appreciated by me and other supporters! ### Gold Sponsors [](https://www.nodion.com) [](https://www.piknik.com) +[](https://www.keepsec.ca) --- diff --git a/docker/Dockerfile.s3tests b/docker/Dockerfile.s3tests index 4b7ea8d92..75c7f5f0a 100644 --- a/docker/Dockerfile.s3tests +++ b/docker/Dockerfile.s3tests @@ -25,7 +25,7 @@ ENV \ NOSETESTS_EXCLUDE="" \ NOSETESTS_ATTR="" \ NOSETESTS_OPTIONS="" \ - S3TEST_CONF="/s3test.conf" + S3TEST_CONF="/s3tests.conf" ENTRYPOINT ["/bin/bash", "-c"] CMD ["sleep 30 && exec ./virtualenv/bin/nosetests ${NOSETESTS_OPTIONS-} ${NOSETESTS_ATTR:+-a $NOSETESTS_ATTR} ${NOSETESTS_EXCLUDE:+-e $NOSETESTS_EXCLUDE}"] @@ -3,13 +3,13 @@ module github.com/seaweedfs/seaweedfs go 1.22.0 require ( - cloud.google.com/go v0.113.0 // indirect - cloud.google.com/go/pubsub v1.38.0 - cloud.google.com/go/storage v1.41.0 + cloud.google.com/go v0.115.0 // indirect + cloud.google.com/go/pubsub v1.40.0 + cloud.google.com/go/storage v1.42.0 github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-storage-blob-go v0.15.0 github.com/Shopify/sarama v1.38.1 - github.com/aws/aws-sdk-go v1.53.5 + github.com/aws/aws-sdk-go v1.55.5 github.com/beorn7/perks v1.0.1 // indirect github.com/bwmarrin/snowflake v0.3.0 github.com/cenkalti/backoff/v4 v4.3.0 @@ -27,9 +27,8 @@ require ( github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4 github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect - github.com/fclairamb/ftpserverlib v0.24.0 + github.com/fclairamb/ftpserverlib v0.24.1 github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/go-redis/redis/v8 v8.11.5 github.com/go-redsync/redsync/v4 v4.13.0 github.com/go-sql-driver/mysql v1.8.1 github.com/go-zookeeper/zk v1.0.3 // indirect @@ -40,7 +39,7 @@ require ( github.com/google/btree v1.1.2 github.com/google/uuid v1.6.0 github.com/google/wire v0.6.0 // indirect - github.com/googleapis/gax-go/v2 v2.12.4 // indirect + github.com/googleapis/gax-go/v2 v2.12.5 // indirect github.com/gorilla/mux v1.8.1 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -53,11 +52,11 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 github.com/karlseguin/ccache/v2 v2.0.8 - github.com/klauspost/compress v1.17.8 // indirect - github.com/klauspost/reedsolomon v1.12.1 + github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/reedsolomon v1.12.3 github.com/kurin/blazer v0.5.3 github.com/lib/pq v1.10.9 - github.com/linxGnu/grocksdb v1.9.1 + github.com/linxGnu/grocksdb v1.9.2 github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-ieproxy v0.0.11 // indirect @@ -71,7 +70,7 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/posener/complete v1.2.3 github.com/pquerna/cachecontrol v0.2.0 - github.com/prometheus/client_golang v1.19.0 + github.com/prometheus/client_golang v1.19.1 github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.15.1 @@ -97,85 +96,86 @@ require ( github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect - go.etcd.io/etcd/client/v3 v3.5.13 - go.mongodb.org/mongo-driver v1.15.0 + go.etcd.io/etcd/client/v3 v3.5.15 + go.mongodb.org/mongo-driver v1.16.0 go.opencensus.io v0.24.0 // indirect - gocloud.dev v0.37.0 - gocloud.dev/pubsub/natspubsub v0.37.0 - gocloud.dev/pubsub/rabbitpubsub v0.37.0 - golang.org/x/crypto v0.24.0 // indirect - golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 - golang.org/x/image v0.16.0 - golang.org/x/net v0.26.0 - golang.org/x/oauth2 v0.20.0 // indirect - golang.org/x/sys v0.21.0 + gocloud.dev v0.38.0 + gocloud.dev/pubsub/natspubsub v0.38.0 + gocloud.dev/pubsub/rabbitpubsub v0.38.0 + golang.org/x/crypto v0.25.0 // indirect + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 + golang.org/x/image v0.18.0 + golang.org/x/net v0.27.0 + golang.org/x/oauth2 v0.21.0 // indirect + golang.org/x/sys v0.22.0 golang.org/x/text v0.16.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d + golang.org/x/tools v0.23.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect - google.golang.org/api v0.181.0 - google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda // indirect - google.golang.org/grpc v1.64.0 - google.golang.org/protobuf v1.34.1 + google.golang.org/api v0.189.0 + google.golang.org/genproto v0.0.0-20240722135656-d784300faade // indirect + google.golang.org/grpc v1.64.1 + google.golang.org/protobuf v1.34.2 gopkg.in/inf.v0 v0.9.1 // indirect modernc.org/b v1.0.0 // indirect - modernc.org/libc v1.50.9 // indirect + modernc.org/libc v1.52.1 // indirect modernc.org/mathutil v1.6.0 modernc.org/memory v1.8.0 // indirect - modernc.org/sqlite v1.30.0 + modernc.org/sqlite v1.30.2 modernc.org/strutil v1.2.0 modernc.org/token v1.1.0 // indirect ) require ( - github.com/Jille/raft-grpc-transport v1.5.0 + github.com/Jille/raft-grpc-transport v1.6.1 github.com/arangodb/go-driver v1.6.2 github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go-v2 v1.27.0 + github.com/aws/aws-sdk-go-v2 v1.30.3 github.com/aws/aws-sdk-go-v2/config v1.27.16 - github.com/aws/aws-sdk-go-v2/credentials v1.17.16 - github.com/aws/aws-sdk-go-v2/service/s3 v1.54.3 + github.com/aws/aws-sdk-go-v2/credentials v1.17.27 + github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2 github.com/cognusion/imaging v1.0.1 github.com/fluent/fluent-logger-golang v1.9.0 - github.com/getsentry/sentry-go v0.28.0 + github.com/getsentry/sentry-go v0.28.1 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50 github.com/hanwen/go-fuse/v2 v2.5.1 - github.com/hashicorp/raft v1.6.1 + github.com/hashicorp/raft v1.7.0 github.com/hashicorp/raft-boltdb/v2 v2.3.0 github.com/orcaman/concurrent-map/v2 v2.0.1 - github.com/parquet-go/parquet-go v0.22.0 - github.com/rabbitmq/amqp091-go v1.9.0 - github.com/rclone/rclone v1.66.0 + github.com/parquet-go/parquet-go v0.23.0 + github.com/rabbitmq/amqp091-go v1.10.0 + github.com/rclone/rclone v1.67.0 github.com/rdleal/intervalst v1.4.0 - github.com/schollz/progressbar/v3 v3.14.3 - github.com/shirou/gopsutil/v3 v3.24.4 + github.com/redis/go-redis/v9 v9.6.1 + github.com/schollz/progressbar/v3 v3.14.4 + github.com/shirou/gopsutil/v3 v3.24.5 github.com/tikv/client-go/v2 v2.0.7 - github.com/ydb-platform/ydb-go-sdk-auth-environ v0.4.2 - github.com/ydb-platform/ydb-go-sdk/v3 v3.68.1 - go.etcd.io/etcd/client/pkg/v3 v3.5.14 + github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0 + github.com/ydb-platform/ydb-go-sdk/v3 v3.76.0 + go.etcd.io/etcd/client/pkg/v3 v3.5.15 go.uber.org/atomic v1.11.0 - golang.org/x/sync v0.7.0 - google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1 + golang.org/x/sync v0.8.0 + google.golang.org/grpc/security/advancedtls v1.0.0 ) require ( - cloud.google.com/go/auth v0.4.1 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect - cloud.google.com/go/compute/metadata v0.3.0 // indirect - cloud.google.com/go/iam v1.1.8 // indirect + cloud.google.com/go/auth v0.7.2 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect + cloud.google.com/go/compute/metadata v0.5.0 // indirect + cloud.google.com/go/iam v1.1.10 // indirect filippo.io/edwards25519 v1.1.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.1.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.2.2 // indirect github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect - github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c // indirect + github.com/ProtonMail/go-crypto v1.0.0 // indirect github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect github.com/ProtonMail/go-srp v0.0.7 // indirect github.com/ProtonMail/gopenpgp/v2 v2.7.4 // indirect @@ -185,22 +185,22 @@ require ( github.com/andybalholm/brotli v1.1.0 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.7 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.7 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.7 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.9 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.9 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.7 // indirect - github.com/aws/aws-sdk-go-v2/service/sns v1.29.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sqs v1.31.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.20.9 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.3 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.28.10 // indirect - github.com/aws/smithy-go v1.20.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 // indirect + github.com/aws/aws-sdk-go-v2/service/sns v1.29.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sqs v1.31.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect + github.com/aws/smithy-go v1.20.3 // indirect github.com/boltdb/bolt v1.3.1 // indirect github.com/bradenaw/juniper v0.15.2 // indirect github.com/buengese/sgzip v0.1.1 // indirect @@ -223,8 +223,8 @@ require ( github.com/flynn/noise v1.0.1 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/geoffgarside/ber v1.1.0 // indirect - github.com/go-chi/chi/v5 v5.0.11 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-chi/chi/v5 v5.0.12 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-resty/resty/v2 v2.11.0 // indirect @@ -248,7 +248,6 @@ require ( github.com/jlaffaye/ftp v0.2.0 // indirect github.com/jonboulle/clockwork v0.3.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/jtolio/eventkit v0.0.0-20231019094657-5d77ebb407d9 // indirect github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect @@ -261,8 +260,8 @@ require ( github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/montanaflynn/stats v0.7.0 // indirect - github.com/nats-io/nats.go v1.34.0 // indirect + github.com/montanaflynn/stats v0.7.1 // indirect + github.com/nats-io/nats.go v1.34.1 // indirect github.com/nats-io/nkeys v0.4.7 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect @@ -270,12 +269,12 @@ require ( github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/onsi/gomega v1.27.10 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/oracle/oci-go-sdk/v65 v65.55.1 // indirect + github.com/oracle/oci-go-sdk/v65 v65.65.3 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect github.com/philhofer/fwd v1.1.2 // indirect - github.com/pierrec/lz4/v4 v4.1.18 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect @@ -290,7 +289,7 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect - github.com/segmentio/encoding v0.3.6 // indirect + github.com/segmentio/encoding v0.4.0 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect github.com/smartystreets/goconvey v1.8.1 // indirect @@ -308,7 +307,7 @@ require ( github.com/twmb/murmur3 v1.1.3 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect - github.com/ydb-platform/ydb-go-genproto v0.0.0-20240316140903-4a47abca1cca // indirect + github.com/ydb-platform/ydb-go-genproto v0.0.0-20240528144234-5d5a685e41f7 // indirect github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect @@ -316,30 +315,31 @@ require ( github.com/zeebo/blake3 v0.2.3 // indirect github.com/zeebo/errs v1.3.0 // indirect go.etcd.io/bbolt v1.3.8 // indirect - go.etcd.io/etcd/api/v3 v3.5.13 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect - go.opentelemetry.io/otel v1.24.0 // indirect - go.opentelemetry.io/otel/metric v1.24.0 // indirect - go.opentelemetry.io/otel/trace v1.24.0 // indirect + go.etcd.io/etcd/api/v3 v3.5.15 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect + go.opentelemetry.io/otel v1.26.0 // indirect + go.opentelemetry.io/otel/metric v1.26.0 // indirect + go.opentelemetry.io/otel/trace v1.26.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/mod v0.17.0 // indirect - golang.org/x/term v0.21.0 // indirect + golang.org/x/mod v0.19.0 // indirect + golang.org/x/term v0.22.0 // indirect golang.org/x/time v0.5.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/validator.v2 v2.0.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect - storj.io/common v0.0.0-20240111121419-ecae1362576c // indirect + storj.io/common v0.0.0-20240424123607-5f226fc92c16 // indirect storj.io/drpc v0.0.33 // indirect + storj.io/eventkit v0.0.0-20240306141230-6cb545e5f892 // indirect storj.io/infectious v0.0.2 // indirect - storj.io/picobuf v0.0.2-0.20230906122608-c4ba17033c6c // indirect - storj.io/uplink v1.12.2 // indirect + storj.io/picobuf v0.0.3 // indirect + storj.io/uplink v1.13.0 // indirect ) // replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft @@ -13,7 +13,6 @@ cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bP cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.63.0/go.mod h1:GmezbQc7T2snqkEXWfZ0sy0VfkB/ivI2DdtJL2DEmlg= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= @@ -37,8 +36,8 @@ cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRY cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= -cloud.google.com/go v0.113.0 h1:g3C70mn3lWfckKBiCVsAshabrDg01pQ0pnX1MNtnMkA= -cloud.google.com/go v0.113.0/go.mod h1:glEqlogERKYeePz6ZdkcLJ28Q2I6aERgDDErBg9GzO8= +cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= +cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= @@ -85,10 +84,10 @@ cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVo cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo= cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0= cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E= -cloud.google.com/go/auth v0.4.1 h1:Z7YNIhlWRtrnKlZke7z3GMqzvuYzdc2z98F9D1NV5Hg= -cloud.google.com/go/auth v0.4.1/go.mod h1:QVBuVEKpCn4Zp58hzRGvL0tjRGU0YqdRTdCHM1IHnro= -cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= -cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= +cloud.google.com/go/auth v0.7.2 h1:uiha352VrCDMXg+yoBtaD0tUF4Kv9vrtrWPYXwutnDE= +cloud.google.com/go/auth v0.7.2/go.mod h1:VEc4p5NNxycWQTMQEDQF0bd6aTMb6VgYDXEwiJJQAbs= +cloud.google.com/go/auth/oauth2adapt v0.2.3 h1:MlxF+Pd3OmSudg/b1yZ5lJwoXCEaeedAguodky1PcKI= +cloud.google.com/go/auth/oauth2adapt v0.2.3/go.mod h1:tMQXOfZzFuNuUxOypHlQEXgdfX5cuhwU+ffUuXRJE8I= cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8= @@ -157,8 +156,8 @@ cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZ cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= -cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= +cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w= @@ -274,8 +273,8 @@ cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQE cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY= cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= -cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0= -cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE= +cloud.google.com/go/iam v1.1.10 h1:ZSAr64oEhQSClwBL670MsJAW5/RLiC6kfw3Bqmd5ZDI= +cloud.google.com/go/iam v1.1.10/go.mod h1:iEgMq62sg8zx446GCaijmA2Miwg5o3UbO+nI47WHJps= cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk= @@ -289,8 +288,8 @@ cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxs cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w= -cloud.google.com/go/kms v1.15.8 h1:szIeDCowID8th2i8XE4uRev5PMxQFqW+JjwYxL9h6xs= -cloud.google.com/go/kms v1.15.8/go.mod h1:WoUHcDjD9pluCg7pNds131awnH429QGvRM3N/4MyoVs= +cloud.google.com/go/kms v1.18.2 h1:EGgD0B9k9tOOkbPhYW1PHo2W0teamAUYMOUIcDRMfPk= +cloud.google.com/go/kms v1.18.2/go.mod h1:YFz1LYrnGsXARuRePL729oINmN5J/5e7nYijgvfiIeY= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= @@ -304,6 +303,8 @@ cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeN cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= +cloud.google.com/go/longrunning v0.5.9 h1:haH9pAuXdPAMqHvzX0zlWQigXT7B0+CL4/2nXXdBo5k= +cloud.google.com/go/longrunning v0.5.9/go.mod h1:HD+0l9/OOW0za6UWdKJtXoFAX/BGg/3Wj8p10NeWF7c= cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM= cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA= @@ -376,8 +377,8 @@ cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjp cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI= cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8= -cloud.google.com/go/pubsub v1.38.0 h1:J1OT7h51ifATIedjqk/uBNPh+1hkvUaH4VKbz4UuAsc= -cloud.google.com/go/pubsub v1.38.0/go.mod h1:IPMJSWSus/cu57UyR01Jqa/bNOQA+XnPF6Z4dKW4fAA= +cloud.google.com/go/pubsub v1.40.0 h1:0LdP+zj5XaPAGtWr2V6r88VXJlmtaB/+fde1q3TU8M0= +cloud.google.com/go/pubsub v1.40.0/go.mod h1:BVJI4sI2FyXp36KFKvFwcfDRDfR8MiLT8mMhmIhdAeA= cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k= cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= @@ -468,8 +469,8 @@ cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= -cloud.google.com/go/storage v1.41.0 h1:RusiwatSu6lHeEXe3kglxakAmAbfV+rhtPqA6i8RBx0= -cloud.google.com/go/storage v1.41.0/go.mod h1:J1WCa/Z2FcgdEDuPUY8DxT5I+d9mFKsCepp5vR6Sq80= +cloud.google.com/go/storage v1.42.0 h1:4QtGpplCVt1wz6g5o1ifXd656P5z+yNgzdw1tVfp0cU= +cloud.google.com/go/storage v1.42.0/go.mod h1:HjMXRFq65pGKFn6hxj6x3HCyR41uSB72Z0SO/Vn6JFQ= cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4= @@ -532,18 +533,18 @@ gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zum git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 h1:U2rTu3Ef+7w9FHKIAXM6ZyqF3UOWJZ12zIm8zECAFfg= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 h1:jBQA3cKT4L2rWMpgE7Yt3Hwh2aUj8KXjIGLxjHeYNNo= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0/go.mod h1:4OG6tQ9EOP/MT0NMjDlRzWoVFxfu9rN9B2X+tlSVktg= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0 h1:1nGuui+4POelzDwI7RG56yfQJHCnKvwfMoU7VsEp+Zg= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0/go.mod h1:99EvauvlcJ1U06amZiksfYz/3aFGyIhWGHVyiZXtBAI= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0 h1:H+U3Gk9zY56G3u872L82bk4thcsy2Gghb9ExT4Zvm1o= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0/go.mod h1:mgrmMSgaLp9hmax62XQTd0N4aAqSE5E0DulSpVYK7vc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0/go.mod h1:T5RfihdXtBDxt1Ch2wobif3TvzTdumDy29kahv6AV9A= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 h1:fXPMAmuh0gDuRDey0atC8cXBuKIlqCzCkL8sm1n9Ov0= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1/go.mod h1:SUZc9YRRHfx2+FAQKNDGrssXehqLpxmwRv2mC/5ntj4= -github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.1.1 h1:pOz+ZkB4uqDgS75Ft6JPeMIsGs5MHz2lkHa1odQ9Ghk= -github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.1.1/go.mod h1:eISpIuAeU2UzR3FO0ODAwvpAnVZsfEISxszPW/chkBw= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 h1:YUUxeiOWgdAQE3pXt2H7QXzZs0q8UBjgRbl56qo8GYM= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2/go.mod h1:dmXQgZuiSubAecswZE+Sm8jkvEa7kQgTPVRvwL/nd0E= +github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.2.2 h1:PmDhkIT8S5U4nkY/s78Xmf7CXT8qCliNEBhbrkBp3Q0= +github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.2.2/go.mod h1:Kj2pCkQ47klX1aAlDnlN/BUvwBiARqIJkc9iw1Up7q8= github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -563,10 +564,10 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mx github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/Jille/raft-grpc-transport v1.5.0 h1:a5c2CVm+Vz3KDhp21vdH6GzA144viOPyG4h2KgS3ufY= -github.com/Jille/raft-grpc-transport v1.5.0/go.mod h1:afVUd8LQKUUo3V/ToLBH3mbSyvivRlMYCDK0eJRGTfQ= +github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/Jille/raft-grpc-transport v1.6.1 h1:gN3sjapb+fVbiebS7AfQQgbV2ecTOI7ur7NPPC7Mhoc= +github.com/Jille/raft-grpc-transport v1.6.1/go.mod h1:HbOjEdu/yzCJ/mjTF6wEOJNbAUpHfU2UOA2hVD4CNFg= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= @@ -583,8 +584,8 @@ github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e h1:lCsqUUACrcM github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e/go.mod h1:Og5/Dz1MiGpCJn51XujZwxiLG7WzvvjE5PRpZBQmAHo= github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE= github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= -github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c h1:kMFnB0vCcX7IL/m9Y5LO+KQYv+t1CQOiFe6+SV2J7bE= -github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= +github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k= github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw= github.com/ProtonMail/go-srp v0.0.7 h1:Sos3Qk+th4tQR64vsxGIxYpN3rdnG9Wf9K4ZloC1JrI= @@ -593,6 +594,7 @@ github.com/ProtonMail/gopenpgp/v2 v2.7.4 h1:Vz/8+HViFFnf2A6XX8JOvZMrA6F5puwNvvF2 github.com/ProtonMail/gopenpgp/v2 v2.7.4/go.mod h1:IhkNEDaxec6NyzSI0PlxapinnwPVIESk8/76da3Ct3g= github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= +github.com/Sereal/Sereal/Go/sereal v0.0.0-20231009093132-b9187f1a92c6/go.mod h1:JwrycNnC8+sZPDyzM3MQ86LvaGzSpfxg885KOOwFRW4= github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A= github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g= github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc= @@ -624,52 +626,50 @@ github.com/arangodb/go-driver v1.6.2 h1:3o4inejwR7VMmsKvQJ6hepx4au9sUT6C/RDrXyku github.com/arangodb/go-driver v1.6.2/go.mod h1:2BCE6y3DNSLqIXnDvf4CR6WdzZZloYudEy+sasimLiQ= github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e h1:Xg+hGrY2LcQBbxd0ZFdbGSyRKTYMZCfBbw/pMJFOk1g= github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e/go.mod h1:mq7Shfa/CaixoDxiyAAc5jZ6CVBAyPaNQCGS7mkj4Ho= -github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= -github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/aws/aws-sdk-go v1.53.5 h1:1OcVWMjGlwt7EU5OWmmEEXqaYfmX581EK317QJZXItM= -github.com/aws/aws-sdk-go v1.53.5/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= -github.com/aws/aws-sdk-go-v2 v1.27.0 h1:7bZWKoXhzI+mMR/HjdMx8ZCC5+6fY0lS5tr0bbgiLlo= -github.com/aws/aws-sdk-go-v2 v1.27.0/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg= +github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= +github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY= +github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 h1:tW1/Rkad38LA15X4UQtjXZXNKsCgkshC3EbmcUmghTg= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3/go.mod h1:UbnqO+zjqk3uIt9yCACHJ9IVNhyhOCnYk8yA19SAWrM= github.com/aws/aws-sdk-go-v2/config v1.27.16 h1:knpCuH7laFVGYTNd99Ns5t+8PuRjDn4HnnZK48csipM= github.com/aws/aws-sdk-go-v2/config v1.27.16/go.mod h1:vutqgRhDUktwSge3hrC3nkuirzkJ4E/mLj5GvI0BQas= -github.com/aws/aws-sdk-go-v2/credentials v1.17.16 h1:7d2QxY83uYl0l58ceyiSpxg9bSbStqBC6BeEeHEchwo= -github.com/aws/aws-sdk-go-v2/credentials v1.17.16/go.mod h1:Ae6li/6Yc6eMzysRL2BXlPYvnrLLBg3D11/AmOjw50k= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.3 h1:dQLK4TjtnlRGb0czOht2CevZ5l6RSyRWAnKeGd7VAFE= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.3/go.mod h1:TL79f2P6+8Q7dTsILpiVST+AL9lkF6PPGI167Ny0Cjw= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.7 h1:lf/8VTF2cM+N4SLzaYJERKEWAXq8MOMpZfU6wEPWsPk= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.7/go.mod h1:4SjkU7QiqK2M9oozyMzfZ/23LmUY+h3oFqhdeP5OMiI= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.7 h1:4OYVp0705xu8yjdyoWix0r9wPIRXnIzzOoUpQVHIJ/g= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.7/go.mod h1:vd7ESTEvI76T2Na050gODNmNU7+OyKrIKroYTu4ABiI= +github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVOpGPgDSi0I9iAP+UI= +github.com/aws/aws-sdk-go-v2/credentials v1.17.27/go.mod h1:gniiwbGahQByxan6YjQUMcW4Aov6bLC3m+evgcoN4r4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 h1:KreluoV8FZDEtI6Co2xuNk/UqI9iwMrOx/87PBNIKqw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11/go.mod h1:SeSUYBLsMYFoRvHE0Tjvn7kbxaUhl75CJi1sbfhMxkU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 h1:SoNJ4RlFEQEbtDcCEt+QG56MY4fm4W8rYirAmq+/DdU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15/go.mod h1:U9ke74k1n2bf+RIgoX1SXFed1HLs51OgUSs+Ph0KJP8= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 h1:C6WHdGnTDIYETAm5iErQUiVNsclNx9qbJVPIt03B6bI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15/go.mod h1:ZQLZqhcu+JhSrA9/NXRm8SkDvsycE+JkV3WGY41e+IM= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.7 h1:/FUtT3xsoHO3cfh+I/kCbcMCN98QZRsiFet/V8QkWSs= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.7/go.mod h1:MaCAgWpGooQoCWZnMur97rGn5dp350w2+CeiV5406wE= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.9 h1:UXqEWQI0n+q0QixzU0yUUQBZXRd5037qdInTIHFTl98= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.9/go.mod h1:xP6Gq6fzGZT8w/ZN+XvGMZ2RU1LeEs7b2yUP5DN8NY4= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.9 h1:Wx0rlZoEJR7JwlSZcHnEa7CNjrSIyVxMFWGAaXy4fJY= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.9/go.mod h1:aVMHdE0aHO3v+f/iw01fmXV/5DbfQ3Bi9nN7nd9bE9Y= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.7 h1:uO5XR6QGBcmPyo2gxofYJLFkcVQ4izOoGDNenlZhTEk= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.7/go.mod h1:feeeAYfAcwTReM6vbwjEyDmiGho+YgBhaFULuXDW8kc= -github.com/aws/aws-sdk-go-v2/service/s3 v1.54.3 h1:57NtjG+WLims0TxIQbjTqebZUKDM03DfM11ANAekW0s= -github.com/aws/aws-sdk-go-v2/service/s3 v1.54.3/go.mod h1:739CllldowZiPPsDFcJHNF4FXrVxaSGVnZ9Ez9Iz9hc= -github.com/aws/aws-sdk-go-v2/service/sns v1.29.2 h1:kHm1SYs/NkxZpKINc4zOXOLJHVMzKtU4d7FlAMtDm50= -github.com/aws/aws-sdk-go-v2/service/sns v1.29.2/go.mod h1:ZIs7/BaYel9NODoYa8PW39o15SFAXDEb4DxOG2It15U= -github.com/aws/aws-sdk-go-v2/service/sqs v1.31.2 h1:A9ihuyTKpS8Z1ou/D4ETfOEFMyokA6JjRsgXWTiHvCk= -github.com/aws/aws-sdk-go-v2/service/sqs v1.31.2/go.mod h1:J3XhTE+VsY1jDsdDY+ACFAppZj/gpvygzC5JE0bTLbQ= -github.com/aws/aws-sdk-go-v2/service/sso v1.20.9 h1:aD7AGQhvPuAxlSUfo0CWU7s6FpkbyykMhGYMvlqTjVs= -github.com/aws/aws-sdk-go-v2/service/sso v1.20.9/go.mod h1:c1qtZUWtygI6ZdvKppzCSXsDOq5I4luJPZ0Ud3juFCA= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.3 h1:Pav5q3cA260Zqez42T9UhIlsd9QeypszRPwC9LdSSsQ= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.3/go.mod h1:9lmoVDVLz/yUZwLaQ676TK02fhCu4+PgRSmMaKR1ozk= -github.com/aws/aws-sdk-go-v2/service/sts v1.28.10 h1:69tpbPED7jKPyzMcrwSvhWcJ9bPnZsZs18NT40JwM0g= -github.com/aws/aws-sdk-go-v2/service/sts v1.28.10/go.mod h1:0Aqn1MnEuitqfsCNyKsdKLhDUOr4txD/g19EfiUqgws= -github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= -github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 h1:Z5r7SycxmSllHYmaAZPpmN8GviDrSGhMS6bldqtXZPw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15/go.mod h1:CetW7bDE00QoGEmPUoZuRog07SGVAUVW6LFpNP0YfIg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17 h1:YPYe6ZmvUfDDDELqEKtAd6bo8zxhkm+XEFEzQisqUIE= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17/go.mod h1:oBtcnYua/CgzCWYN7NZ5j7PotFDaFSUjCYVTtfyn7vw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 h1:HGErhhrxZlQ044RiM+WdoZxp0p+EGM62y3L6pwA4olE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17/go.mod h1:RkZEx4l0EHYDJpWppMJ3nD9wZJAa8/0lq9aVC+r2UII= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 h1:246A4lSTXWJw/rmlQI+TT2OcqeDMKBdyjEQrafMaQdA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15/go.mod h1:haVfg3761/WF7YPuJOER2MP0k4UAXyHaLclKXB6usDg= +github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2 h1:sZXIzO38GZOU+O0C+INqbH7C2yALwfMWpd64tONS/NE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2/go.mod h1:Lcxzg5rojyVPU/0eFwLtcyTaek/6Mtic5B1gJo7e/zE= +github.com/aws/aws-sdk-go-v2/service/sns v1.29.4 h1:VhW/J21SPH9bNmk1IYdZtzqA6//N2PB5Py5RexNmLVg= +github.com/aws/aws-sdk-go-v2/service/sns v1.29.4/go.mod h1:DojKGyWXa4p+e+C+GpG7qf02QaE68Nrg2v/UAXQhKhU= +github.com/aws/aws-sdk-go-v2/service/sqs v1.31.4 h1:mE2ysZMEeQ3ulHWs4mmc4fZEhOfeY1o6QXAfDqjbSgw= +github.com/aws/aws-sdk-go-v2/service/sqs v1.31.4/go.mod h1:lCN2yKnj+Sp9F6UzpoPPTir+tSaC9Jwf6LcmTqnXFZw= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 h1:BXx0ZIxvrJdSgSvKTZ+yRBeSqqgPM89VPlulEcl37tM= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.4/go.mod h1:ooyCOXjvJEsUw7x+ZDHeISPMhtwI3ZCB7ggFMcFfWLU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 h1:yiwVzJW2ZxZTurVbYWA7QOrAaCYQR72t0wrSBfoesUE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4/go.mod h1:0oxfLkpz3rQ/CHlx5hB7H69YUpFiI1tql6Q6Ne+1bCw= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 h1:ZsDKRLXGWHk8WdtyYMoGNO7bTudrvuKpDKgMVRlepGE= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.3/go.mod h1:zwySh8fpFyXp9yOr/KVzxOl8SRqgf/IDw5aUt9UKFcQ= +github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE= +github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -685,6 +685,10 @@ github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/bradenaw/juniper v0.15.2 h1:0JdjBGEF2jP1pOxmlNIrPhAoQN7Ng5IMAY5D0PHMW4U= github.com/bradenaw/juniper v0.15.2/go.mod h1:UX4FX57kVSaDp4TPqvSjkAAewmRFAfXf27BOs5z9dq8= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/buengese/sgzip v0.1.1 h1:ry+T8l1mlmiWEsDrH/YHZnCVWD2S3im1KLsyO+8ZmTU= github.com/buengese/sgzip v0.1.1/go.mod h1:i5ZiXGF3fhV7gL1xaRRL1nDnmpNj0X061FQzOS8VMas= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= @@ -750,6 +754,9 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892/go.mod h1:CTDl0pzVzE5DEzZhPfvhY/9sPFMQIxaJ9VAMs9AagrE= +github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc= +github.com/dgryski/go-ddmin v0.0.0-20210904190556-96a6d69f1034/go.mod h1:zz4KxBkcXUWKjIcrc+uphJ1gPh/t18ymGm3PmQ+VGTk= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= @@ -801,13 +808,11 @@ github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4 h1:0YtRCqIZs2+Tz4 github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4/go.mod h1:vsJz7uE339KUCpBXx3JAJzSRH7Uk4iGGyJzR529qDIA= github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= -github.com/fclairamb/ftpserverlib v0.24.0 h1:c0TaueBw41pzn3VS5G9+xFQMYEl50X0ymiDCnKHLHRE= -github.com/fclairamb/ftpserverlib v0.24.0/go.mod h1:+9afJRWESpCq4/O8Vr00Q2jfinRxP6PiCpXph6CgGuc= +github.com/fclairamb/ftpserverlib v0.24.1 h1:D+dDP+KibZKI182zQlITMJmaayCyIYpLpprzg8ZhtqA= +github.com/fclairamb/ftpserverlib v0.24.1/go.mod h1:aAwyOAC6IIe+IZeeGD1QjuE3GGDzqW/c5Xtn+Dp0JUM= github.com/fclairamb/go-log v0.5.0 h1:Gz9wSamEaA6lta4IU2cjJc2xSq5sV5VYSB5w/SUHhVc= github.com/fclairamb/go-log v0.5.0/go.mod h1:XoRO1dYezpsGmLLkZE9I+sHqpqY65p8JA+Vqblb7k40= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -831,15 +836,15 @@ github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uq github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w= github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= -github.com/getsentry/sentry-go v0.28.0 h1:7Rqx9M3ythTKy2J6uZLHmc8Sz9OGgIlseuO1iBX/s0M= -github.com/getsentry/sentry-go v0.28.0/go.mod h1:1fQZ+7l7eeJ3wYi82q5Hg8GqAPgefRq+FP/QhafYVgg= +github.com/getsentry/sentry-go v0.28.1 h1:zzaSm/vHmGllRM6Tpx1492r0YDzauArdBfkJRtY6P5k= +github.com/getsentry/sentry-go v0.28.1/go.mod h1:1fQZ+7l7eeJ3wYi82q5Hg8GqAPgefRq+FP/QhafYVgg= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= -github.com/go-chi/chi/v5 v5.0.11 h1:BnpYbFZ3T3S1WMpD79r7R5ThWX40TaFB7L31Y8xqSwA= -github.com/go-chi/chi/v5 v5.0.11/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= +github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= @@ -861,8 +866,8 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= @@ -1040,8 +1045,8 @@ github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= -github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw4Z96qg= -github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI= +github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBYGmXdxA= +github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= @@ -1065,16 +1070,13 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v1.6.2 h1:NOtoftovWkDheyUM/8JW3QMiXyxJK3uHRK7wV04nD2I= github.com/hashicorp/go-hclog v1.6.2/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-msgpack v1.1.5 h1:9byZdVjKTe5mce63pRVNP1L7UAmdHOTEMGehn6KvJWs= -github.com/hashicorp/go-msgpack v1.1.5/go.mod h1:gWVc3sv/wbDmR3rQsj1CAktEZzoz1YNK9NfGLXJ69/4= github.com/hashicorp/go-msgpack/v2 v2.1.1 h1:xQEY9yB2wnHitoSzk/B9UjXWRQ67QKu5AOm8aFp8N3I= github.com/hashicorp/go-msgpack/v2 v2.1.1/go.mod h1:upybraOAblm4S7rx0+jeNy+CWWhzywQsSRV5033mMu4= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= @@ -1094,9 +1096,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/raft v1.3.7/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= -github.com/hashicorp/raft v1.6.1 h1:v/jm5fcYHvVkL0akByAp+IDdDSzCNCGhdO6VdB56HIM= -github.com/hashicorp/raft v1.6.1/go.mod h1:N1sKh6Vn47mrWvEArQgILTyng8GoDRNYlgKyK7PMjs0= +github.com/hashicorp/raft v1.7.0 h1:4u24Qn6lQ6uwziM++UgsyiT64Q8GyRn43CV41qPiz1o= +github.com/hashicorp/raft v1.7.0/go.mod h1:N1sKh6Vn47mrWvEArQgILTyng8GoDRNYlgKyK7PMjs0= github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702 h1:RLKEcCuKcZ+qp2VlaaZsYZfLOmIiuJNpEi48Rl8u9cQ= github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702/go.mod h1:nTakvJ4XYq45UXtn0DbwR4aU9ZdjlnIenpbs6Cd+FM0= github.com/hashicorp/raft-boltdb/v2 v2.3.0 h1:fPpQR1iGEVYjZ2OELvUHX600VAK5qmdnDEv3eXOwZUA= @@ -1143,8 +1144,6 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/jtolio/eventkit v0.0.0-20231019094657-5d77ebb407d9 h1:5aw4qPuPiOjYJGATzHQllDfom77cgZrzE1ttOj7GRWQ= -github.com/jtolio/eventkit v0.0.0-20231019094657-5d77ebb407d9/go.mod h1:PXFUrknJu7TkBNyL8t7XWDPtDFFLFrNQQAdsXv9YfJE= github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 h1:JcltaO1HXM5S2KYOYcKgAV7slU0xPy1OcvrVgn98sRQ= github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7/go.mod h1:MEkhEPFwP3yudWO0lj6vfYpLIB+3eIcuIW+e0AZzUQk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= @@ -1162,14 +1161,14 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= -github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/klauspost/reedsolomon v1.12.1 h1:NhWgum1efX1x58daOBGCFWcxtEhOhXKKl1HAPQUp03Q= -github.com/klauspost/reedsolomon v1.12.1/go.mod h1:nEi5Kjb6QqtbofI6s+cbG/j1da11c96IBYBSnVGtuBs= +github.com/klauspost/reedsolomon v1.12.3 h1:tzUznbfc3OFwJaTebv/QdhnFf2Xvb7gZ24XaHLBPmdc= +github.com/klauspost/reedsolomon v1.12.3/go.mod h1:3K5rXwABAvzGeR01r6pWZieUALXO/Tq7bFKGIb4m4WI= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koofr/go-httpclient v0.0.0-20230225102643-5d51a2e9dea6 h1:uF5FHZ/L5gvZTyBNhhcm55rRorL66DOs4KIeeVXZ8eI= github.com/koofr/go-httpclient v0.0.0-20230225102643-5d51a2e9dea6/go.mod h1:6HAT62hK6QH+ljNtZayJCKpbZy5hJIB12+1Ze1bFS7M= @@ -1196,9 +1195,8 @@ github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/linxGnu/grocksdb v1.9.1 h1:LmwuHzsWglxJrIES9jvS2O1xTPD2nnKYhAQDx5dIyRo= -github.com/linxGnu/grocksdb v1.9.1/go.mod h1:QYiYypR2d4v63Wj1adOOfzglnoII0gLj3PNh4fZkcFA= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/linxGnu/grocksdb v1.9.2 h1:O3mzvO0wuzQ9mtlHbDrShixyVjVbmuqTjFrzlf43wZ8= +github.com/linxGnu/grocksdb v1.9.2/go.mod h1:QYiYypR2d4v63Wj1adOOfzglnoII0gLj3PNh4fZkcFA= github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed h1:036IscGBfJsFIgJQzlui7nK1Ncm0tp2ktmPj8xO4N/0= github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= @@ -1208,8 +1206,6 @@ github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0V github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -1217,8 +1213,6 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-ieproxy v0.0.11 h1:MQ/5BuGSgDAHZOJe6YY80IF2UVCfGkwfo6AeD7HtHYo= github.com/mattn/go-ieproxy v0.0.11/go.mod h1:/NsJd+kxZBmjMc5hrJCKMbP57B84rvq9BiDRbtO9AS0= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -1250,15 +1244,15 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/montanaflynn/stats v0.7.0 h1:r3y12KyNxj/Sb/iOE46ws+3mS1+MZca1wlHQFPsY/JU= -github.com/montanaflynn/stats v0.7.0/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt/v2 v2.5.0 h1:WQQ40AAlqqfx+f6ku+i0pOVm+ASirD4fUh+oQsiE9Ak= github.com/nats-io/jwt/v2 v2.5.0/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI= github.com/nats-io/nats-server/v2 v2.9.23 h1:6Wj6H6QpP9FMlpCyWUaNu2yeZ/qGj+mdRkZ1wbikExU= github.com/nats-io/nats-server/v2 v2.9.23/go.mod h1:wEjrEy9vnqIGE4Pqz4/c75v9Pmaq7My2IgFmnykc4C0= -github.com/nats-io/nats.go v1.34.0 h1:fnxnPCNiwIG5w08rlMcEKTUw4AV/nKyGCOJE8TdhSPk= -github.com/nats-io/nats.go v1.34.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= +github.com/nats-io/nats.go v1.34.1 h1:syWey5xaNHZgicYBemv0nohUPPmaLteiBEUT6Q5+F/4= +github.com/nats-io/nats.go v1.34.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= @@ -1285,12 +1279,12 @@ github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3ev github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/oracle/oci-go-sdk/v65 v65.55.1 h1:82j4YHtqeJJiOPyLKxP4/x3Oi8tv2o1etKMziY7yM7o= -github.com/oracle/oci-go-sdk/v65 v65.55.1/go.mod h1:IBEV9l1qBzUpo7zgGaRUhbB05BVfcDGYRFBCPlTcPp0= +github.com/oracle/oci-go-sdk/v65 v65.65.3 h1:Vx2MbWaXlqYW821SJoZgZM7FTzaVWW9S5QHiamD5+ng= +github.com/oracle/oci-go-sdk/v65 v65.65.3/go.mod h1:IBEV9l1qBzUpo7zgGaRUhbB05BVfcDGYRFBCPlTcPp0= github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c= github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM= -github.com/parquet-go/parquet-go v0.22.0 h1:9G32efs+11L/MDc0Zt05AuvBubRGAp5lRKufv6pB/B8= -github.com/parquet-go/parquet-go v0.22.0/go.mod h1:3VBP+djJCNuV+D5uSUs2pWQufk2yKO+9pwYvXglsB8Y= +github.com/parquet-go/parquet-go v0.23.0 h1:dyEU5oiHCtbASyItMCD2tXtT2nPmoPbKpqf0+nnGrmk= +github.com/parquet-go/parquet-go v0.23.0/go.mod h1:MnwbUcFHU6uBYMymKAlPPAw9yh3kE1wWl6Gl1uLdkNk= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= @@ -1307,8 +1301,8 @@ github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2 github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= -github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= @@ -1339,17 +1333,16 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.2.3 h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXqo= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig= github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/pquerna/cachecontrol v0.2.0 h1:vBXSNuE5MYP9IJ5kjsdo8uq+w41jSPgvba2DEnkRx9k= github.com/pquerna/cachecontrol v0.2.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= +github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= -github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -1357,13 +1350,11 @@ github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6T github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= @@ -1374,16 +1365,16 @@ github.com/quic-go/qtls-go1-20 v0.4.1 h1:D33340mCNDAIKBqXuAvexTNMUByrYmFYVfKfDN5 github.com/quic-go/qtls-go1-20 v0.4.1/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= github.com/quic-go/quic-go v0.40.1 h1:X3AGzUNFs0jVuO3esAGnTfvdgvL4fq655WaOi1snv1Q= github.com/quic-go/quic-go v0.40.1/go.mod h1:PeN7kuVJ4xZbxSv/4OX6S1USOX8MJvydwpTx31vx60c= -github.com/rabbitmq/amqp091-go v1.9.0 h1:qrQtyzB4H8BQgEuJwhmVQqVHB9O4+MNDJCCAcpc3Aoo= -github.com/rabbitmq/amqp091-go v1.9.0/go.mod h1:+jPrT9iY2eLjRaMSRHUhc3z14E/l85kv/f+6luSD3pc= -github.com/rclone/rclone v1.66.0 h1:w7HVJrdiUf9VVmmv0Q0RopxN9wvEwpu673db5PhTeFA= -github.com/rclone/rclone v1.66.0/go.mod h1:EA+fFWNjsawclx5rV7ZkZU2Z0WIgvUPesjhUrG8yVJo= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/rclone/rclone v1.67.0 h1:yLRNgHEG2vQ60HCuzFqd0hYwKCRuWuvPUhvhMJ2jI5E= +github.com/rclone/rclone v1.67.0/go.mod h1:Cb3Ar47M/SvwfhAjZTbVXdtrP/JLtPFCq2tkdtBVC6w= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rdleal/intervalst v1.4.0 h1:DWrcNJ9kxPOYnXxExrQkta1xsKFRN0rTOVXkBDJzVYg= github.com/rdleal/intervalst v1.4.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ= -github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= -github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= +github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4= +github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA= github.com/redis/rueidis v1.0.19 h1:s65oWtotzlIFN8eMPhyYwxlwLR1lUdhza2KtWprKYSo= github.com/redis/rueidis v1.0.19/go.mod h1:8B+r5wdnjwK3lTFml5VtxjzGOQAC+5UmujoD12pDrEo= github.com/rekby/fixenv v0.3.2/go.mod h1:/b5LRc06BYJtslRtHKxsPWFT/ySpHV+rWvzTg+XWk4c= @@ -1411,20 +1402,19 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/schollz/progressbar/v3 v3.14.3 h1:oOuWW19ka12wxYU1XblR4n16wF/2Y1dBLMarMo6p4xU= -github.com/schollz/progressbar/v3 v3.14.3/go.mod h1:aT3UQ7yGm+2ZjeXPqsjTenwL3ddUiuZ0kfQ/2tHlyNI= +github.com/schollz/progressbar/v3 v3.14.4 h1:W9ZrDSJk7eqmQhd3uxFNNcTr0QL+xuGNI9dEMrw0r74= +github.com/schollz/progressbar/v3 v3.14.4/go.mod h1:aT3UQ7yGm+2ZjeXPqsjTenwL3ddUiuZ0kfQ/2tHlyNI= github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30= github.com/seaweedfs/goexif v1.0.3/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk= github.com/seaweedfs/raft v1.1.3 h1:5B6hgneQ7IuU4Ceom/f6QUt8pEeqjcsRo+IxlyPZCws= github.com/seaweedfs/raft v1.1.3/go.mod h1:9cYlEBA+djJbnf/5tWsCybtbL7ICYpi+Uxcg3MxjuNs= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= -github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= -github.com/segmentio/encoding v0.3.6 h1:E6lVLyDPseWEulBmCmAKPanDd3jiyGDo5gMcugCRwZQ= -github.com/segmentio/encoding v0.3.6/go.mod h1:n0JeuIqEQrQoPDGsjo8UNd1iA0U8d8+oHAA4E3G3OxM= +github.com/segmentio/encoding v0.4.0 h1:MEBYvRqiUB2nfR2criEXWqwdY6HJOUrCn5hboVOVmy8= +github.com/segmentio/encoding v0.4.0/go.mod h1:/d03Cd8PoaDeceuhUUUQWjU0KhWjrmYrWPgtJHYZSnI= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU= -github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8= +github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= +github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= @@ -1476,6 +1466,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= @@ -1502,10 +1493,8 @@ github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 h1:bzlSSzw+6qTwPs8p github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1/go.mod h1:3cTcfo8GRA2H/uSttqA3LvMfMSHVBJaXk3IgkFXFVxo= github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= -github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4= github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0= -github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr4= github.com/tklauser/numcpus v0.7.0/go.mod h1:bb6dMVcj8A42tSE7i32fsIUCbQNllK5iDguyOZRUzAY= github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365 h1:6iRwZdrFUzbcVYZwa8dXTIILGIxmmhjyUPJEcwzPGaU= @@ -1541,14 +1530,14 @@ github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e h1:9LPdmD github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE= github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I= github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I= -github.com/ydb-platform/ydb-go-genproto v0.0.0-20240316140903-4a47abca1cca h1:PliQWLwi2gTSOk7QyYQ9GfjvvikmibLWmaplKHy+kfo= -github.com/ydb-platform/ydb-go-genproto v0.0.0-20240316140903-4a47abca1cca/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I= -github.com/ydb-platform/ydb-go-sdk-auth-environ v0.4.2 h1:qsF9h/XDaMqriPPRloe6tCi4b7jIeWqUTQy+J26PwiQ= -github.com/ydb-platform/ydb-go-sdk-auth-environ v0.4.2/go.mod h1:feJz7wSEf53OKMxZDoYRxsMQt123fEv8uEaXyUYkwMg= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20240528144234-5d5a685e41f7 h1:nL8XwD6fSst7xFUirkaWJmE7kM0CdWRYgu6+YQer1d4= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20240528144234-5d5a685e41f7/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I= +github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0 h1:/NyPd9KnCJgzrEXCArqk1ThqCH2Dh31uUwl88o/VkuM= +github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0/go.mod h1:9YzkhlIymWaJGX6KMU3vh5sOf3UKbCXkG/ZdjaI3zNM= github.com/ydb-platform/ydb-go-sdk/v3 v3.44.0/go.mod h1:oSLwnuilwIpaF5bJJMAofnGgzPJusoI3zWMNb8I+GnM= github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0= -github.com/ydb-platform/ydb-go-sdk/v3 v3.68.1 h1:3ypB8s1VENsKKQ9oAmoVQOtE6fEUruEn7iJ343qQY0o= -github.com/ydb-platform/ydb-go-sdk/v3 v3.68.1/go.mod h1:C1QR3y5bK6IKV7/TyMgQHzCvbD9PMtCGonBHE//HkbA= +github.com/ydb-platform/ydb-go-sdk/v3 v3.76.0 h1:McLmV3vxJDBAjr7nqKqMtvxfMToGse1Uh6DQ3QAGYqQ= +github.com/ydb-platform/ydb-go-sdk/v3 v3.76.0/go.mod h1:QMmT1fMKZnpid73USXLJawh+32bKySSE2WtEnBUIKd8= github.com/ydb-platform/ydb-go-yc v0.12.1 h1:qw3Fa+T81+Kpu5Io2vYHJOwcrYrVjgJlT6t/0dOXJrA= github.com/ydb-platform/ydb-go-yc v0.12.1/go.mod h1:t/ZA4ECdgPWjAb4jyDe8AzQZB5dhpGbi3iCahFaNwBY= github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 h1:9E5q8Nsy2RiJMZDNVy0A3KUrIMBPakJ2VgloeWbcI84= @@ -1581,14 +1570,14 @@ go.einride.tech/aip v0.67.1 h1:d/4TW92OxXBngkSOwWS2CH5rez869KpKMaN44mdxkFI= go.einride.tech/aip v0.67.1/go.mod h1:ZGX4/zKw8dcgzdLsrvpOOGxfxI2QSk12SlP7d6c0/XI= go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.etcd.io/etcd/api/v3 v3.5.13 h1:8WXU2/NBge6AUF1K1gOexB6e07NgsN1hXK0rSTtgSp4= -go.etcd.io/etcd/api/v3 v3.5.13/go.mod h1:gBqlqkcMMZMVTMm4NDZloEVJzxQOQIls8splbqBDa0c= -go.etcd.io/etcd/client/pkg/v3 v3.5.14 h1:SaNH6Y+rVEdxfpA2Jr5wkEvN6Zykme5+YnbCkxvuWxQ= -go.etcd.io/etcd/client/pkg/v3 v3.5.14/go.mod h1:8uMgAokyG1czCtIdsq+AGyYQMvpIKnSvPjFMunkgeZI= -go.etcd.io/etcd/client/v3 v3.5.13 h1:o0fHTNJLeO0MyVbc7I3fsCf6nrOqn5d+diSarKnB2js= -go.etcd.io/etcd/client/v3 v3.5.13/go.mod h1:cqiAeY8b5DEEcpxvgWKsbLIWNM/8Wy2xJSDMtioMcoI= -go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc= -go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= +go.etcd.io/etcd/api/v3 v3.5.15 h1:3KpLJir1ZEBrYuV2v+Twaa/e2MdDCEZ/70H+lzEiwsk= +go.etcd.io/etcd/api/v3 v3.5.15/go.mod h1:N9EhGzXq58WuMllgH9ZvnEr7SI9pS0k0+DHZezGp7jM= +go.etcd.io/etcd/client/pkg/v3 v3.5.15 h1:fo0HpWz/KlHGMCC+YejpiCmyWDEuIpnTDzpJLB5fWlA= +go.etcd.io/etcd/client/pkg/v3 v3.5.15/go.mod h1:mXDI4NAOwEiszrHCb0aqfAYNCrZP4e9hRca3d1YK8EU= +go.etcd.io/etcd/client/v3 v3.5.15 h1:23M0eY4Fd/inNv1ZfU3AxrbbOdW79r9V9Rl62Nm6ip4= +go.etcd.io/etcd/client/v3 v3.5.15/go.mod h1:CLSJxrYjvLtHsrPKsy7LmZEE+DK2ktfd2bN4RhBMwlU= +go.mongodb.org/mongo-driver v1.16.0 h1:tpRsfBJMROVHKpdGyc1BBEzzjDUWjItxbVSZ8Ls4BQ4= +go.mongodb.org/mongo-driver v1.16.0/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -1598,18 +1587,18 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= -go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= -go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= -go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= -go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0 h1:A3SayB3rNyt+1S6qpI9mHPkeHTZbD7XILEqWnYZb2l0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0/go.mod h1:27iA5uvhuRNmalO+iEUdVn5ZMj2qy10Mm+XRIpRmyuU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 h1:Xs2Ncz0gNihqu9iosIZ5SkBbWo5T8JhhLJFMQL1qmLI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0/go.mod h1:vy+2G/6NvVMpwGX/NyLqcC41fxepnuKHk16E6IZUcJc= +go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= +go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= +go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= +go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= -go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= -go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= +go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= @@ -1621,7 +1610,6 @@ go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= @@ -1635,12 +1623,12 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -gocloud.dev v0.37.0 h1:XF1rN6R0qZI/9DYjN16Uy0durAmSlf58DHOcb28GPro= -gocloud.dev v0.37.0/go.mod h1:7/O4kqdInCNsc6LqgmuFnS0GRew4XNNYWpA44yQnwco= -gocloud.dev/pubsub/natspubsub v0.37.0 h1:pnszLOYtyOkwB8oqiwYA11OcJpZ0MGJYBhnKl8c3tCs= -gocloud.dev/pubsub/natspubsub v0.37.0/go.mod h1:EN6ORFOviXxR/KALVQjTNh34EUcYQzMke6BCC6Czb8g= -gocloud.dev/pubsub/rabbitpubsub v0.37.0 h1:vNmxM7C+6O6WbErN0TBB8NtrrUFHq3OS0yfokGVsJhQ= -gocloud.dev/pubsub/rabbitpubsub v0.37.0/go.mod h1:f3FInA6G/zOhFoTPLZKca1CH3WknrJEd0Nlryl2C6fw= +gocloud.dev v0.38.0 h1:SpxfaOc/Fp4PeO8ui7wRcCZV0EgXZ+IWcVSLn6ZMSw0= +gocloud.dev v0.38.0/go.mod h1:3XjKvd2E5iVNu/xFImRzjN0d/fkNHe4s0RiKidpEUMQ= +gocloud.dev/pubsub/natspubsub v0.38.0 h1:sJDcxIc6N+VsYJxPl5hNs6V/mLTGOwgS3EZMlawsYSg= +gocloud.dev/pubsub/natspubsub v0.38.0/go.mod h1:JWVrD4tC7AiZYw+TXvMQM37Z7Clc3UCB7kXVdDoMIr0= +gocloud.dev/pubsub/rabbitpubsub v0.38.0 h1:pyXHhQi+uPr6BYucdydHZ7QWEIf/eeXjCAC8igkH7xs= +gocloud.dev/pubsub/rabbitpubsub v0.38.0/go.mod h1:U5BTX9xKy76RR44r8BJH5qkcNYA+3cBcazx5XTVVuHM= golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1665,8 +1653,8 @@ golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1682,8 +1670,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= -golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu+kQvCqcsoVaQgSV60o= -golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1697,8 +1685,8 @@ golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeap golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= -golang.org/x/image v0.16.0 h1:9kloLAKhUufZhA12l5fwnx2NZW39/we1UhBesW433jw= -golang.org/x/image v0.16.0/go.mod h1:ugSZItdV4nOxyqp56HmXwH0Ry0nBCpjnZdpDaIHdoPs= +golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= +golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1729,14 +1717,14 @@ golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -1775,7 +1763,6 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210907225631-ff17edfbf26d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -1802,10 +1789,11 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1834,8 +1822,8 @@ golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= -golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= -golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1853,16 +1841,16 @@ golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1873,7 +1861,6 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1918,12 +1905,10 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1953,14 +1938,12 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1975,8 +1958,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2013,7 +1996,6 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190424220101-1e8e1cfdf96b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -2051,7 +2033,6 @@ golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200806022845-90696ccdc692/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -2074,9 +2055,10 @@ golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= +golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2150,8 +2132,8 @@ google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/ google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= -google.golang.org/api v0.181.0 h1:rPdjwnWgiPPOJx3IcSAQ2III5aX5tCer6wMpa/xmZi4= -google.golang.org/api v0.181.0/go.mod h1:MnQ+M0CFsfUwA5beZ+g/vCBCPXvtmZwRz2qzZk8ih1k= +google.golang.org/api v0.189.0 h1:equMo30LypAkdkLMBqfeIqtyAnlyig1JSZArl4XPwdI= +google.golang.org/api v0.189.0/go.mod h1:FLWGJKb0hb+pU2j+rJqwbnsF+ym+fQs73rbJ+KAUgy8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -2189,7 +2171,6 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -2286,12 +2267,12 @@ google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA= google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= -google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda h1:wu/KJm9KJwpfHWhkkZGohVC6KRrc1oJNr4jwtQMOQXw= -google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda/go.mod h1:g2LLCvCeCSir/JJSWosk19BR4NVxGqHUC6rxIRsd7Aw= -google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae h1:AH34z6WAGVNkllnKs5raNq3yRq93VnjBG6rpfub/jYk= -google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae/go.mod h1:FfiGhwUm6CJviekPrc0oJ+7h29e+DmWU6UtjX0ZvI7Y= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8 h1:mxSlqyb8ZAHsYDCfiXN1EDdNTdvjUJSLY+OnAUtYNYA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8/go.mod h1:I7Y+G38R2bu5j1aLzfFmQfTcU/WnFuqDwLZAbvKTKpM= +google.golang.org/genproto v0.0.0-20240722135656-d784300faade h1:lKFsS7wpngDgSCeFn7MoLy+wBDQZ1UQIJD4UNM1Qvkg= +google.golang.org/genproto v0.0.0-20240722135656-d784300faade/go.mod h1:FfBgJBJg9GcpPvKIuHSZ/aE1g2ecGL74upMzGZjiGEY= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade h1:oCRSWfwGXQsqlVdErcyTt4A93Y8fo0/9D4b1gnI++qo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -2332,13 +2313,13 @@ google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsA google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= -google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= -google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= +google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b h1:NuxyvVZoDfHZwYW9LD4GJiF5/nhiSyP4/InTrvw9Ibk= google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b/go.mod h1:IBqQ7wSUJ2Ep09a8rMWFsg4fmI2r38zwsq8a0GgxXpM= -google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1 h1:0emxaJWaG6CfrA9Nbe4aHWbFz5AXw2QPEJP0/f42LCE= -google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1/go.mod h1:PoKncN6QA5h/eFRzlCWpHSZnXF2pCtnBzAfeanB8OGQ= +google.golang.org/grpc/security/advancedtls v1.0.0 h1:/KQ7VP/1bs53/aopk9QhuPyFAp9Dm9Ejix3lzYkCrDA= +google.golang.org/grpc/security/advancedtls v1.0.0/go.mod h1:o+s4go+e1PJ2AjuQMY5hU82W7lDlefjJA6FqEHRVHWk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -2355,8 +2336,8 @@ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -2369,6 +2350,7 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= @@ -2376,6 +2358,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/validator.v2 v2.0.1 h1:xF0KWyGWXm/LM2G1TrEjqOu4pa6coO9AlWSf3msVfDY= gopkg.in/validator.v2 v2.0.1/go.mod h1:lIUZBlB3Im4s/eYp39Ry/wkR02yOPhZ9IwIRBjuPuG8= +gopkg.in/vmihailenco/msgpack.v2 v2.9.2/go.mod h1:/3Dn1Npt9+MYyLpYYXjInO/5jvMLamn+AEGwNEOatn8= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -2412,8 +2395,8 @@ modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWs modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws= modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo= -modernc.org/ccgo/v4 v4.17.8 h1:yyWBf2ipA0Y9GGz/MmCmi3EFpKgeS7ICrAFes+suEbs= -modernc.org/ccgo/v4 v4.17.8/go.mod h1:buJnJ6Fn0tyAdP/dqePbrrvLyr6qslFfTbFrCuaYvtA= +modernc.org/ccgo/v4 v4.17.10 h1:6wrtRozgrhCxieCeJh85QsxkX/2FFrT9hdaWPlbn4Zo= +modernc.org/ccgo/v4 v4.17.10/go.mod h1:0NBHgsqTTpm9cA5z2ccErvGZmtntSM9qD2kFAs6pjXM= modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= @@ -2429,8 +2412,8 @@ modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU= modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0= modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s= -modernc.org/libc v1.50.9 h1:hIWf1uz55lorXQhfoEoezdUHjxzuO6ceshET/yWjSjk= -modernc.org/libc v1.50.9/go.mod h1:15P6ublJ9FJR8YQCGy8DeQ2Uwur7iW9Hserr/T3OFZE= +modernc.org/libc v1.52.1 h1:uau0VoiT5hnR+SpoWekCKbLqm7v6dhRL3hI+NQhgN3M= +modernc.org/libc v1.52.1/go.mod h1:HR4nVzFDSDizP620zcMCgjb1/8xk2lg5p/8yjfGv1IQ= modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= @@ -2448,8 +2431,8 @@ modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4= -modernc.org/sqlite v1.30.0 h1:8YhPUs/HTnlEgErn/jSYQTwHN/ex8CjHHjg+K9iG7LM= -modernc.org/sqlite v1.30.0/go.mod h1:cgkTARJ9ugeXSNaLBPK3CqbOe7Ec7ZhWPoMFGldEYEw= +modernc.org/sqlite v1.30.2 h1:IPVVkhLu5mMVnS1dQgh3h0SAACRWcVk7aoLP9Us3UCk= +modernc.org/sqlite v1.30.2/go.mod h1:DUmsiWQDaAvU4abhc/N+djlom/L2o8f7gZ95RCvyoLU= modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= @@ -2464,13 +2447,15 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -storj.io/common v0.0.0-20240111121419-ecae1362576c h1:pdNDgFEX2EWmjZLflwQUPKGsq8GYcrdUhihHzgSkutw= -storj.io/common v0.0.0-20240111121419-ecae1362576c/go.mod h1:s3e2Lyg2jCwqL9TLFmesDgoZBPQIByBiKZ+AJOI9qvc= +storj.io/common v0.0.0-20240424123607-5f226fc92c16 h1:Eg2S+KnMD7yCXnlxG5uHUrGOO9H7o1v4Fl/175n7tWc= +storj.io/common v0.0.0-20240424123607-5f226fc92c16/go.mod h1:MFl009RHY4tIqySVNy/6EmgRw2q60d26h9N/nb7JxGU= storj.io/drpc v0.0.33 h1:yCGZ26r66ZdMP0IcTYsj7WDAUIIjzXk6DJhbhvt9FHI= storj.io/drpc v0.0.33/go.mod h1:vR804UNzhBa49NOJ6HeLjd2H3MakC1j5Gv8bsOQT6N4= +storj.io/eventkit v0.0.0-20240306141230-6cb545e5f892 h1:IVzNtTR1VT+QNALCNLX/Z+0hzGo/Y2XI7umgG5PFwOk= +storj.io/eventkit v0.0.0-20240306141230-6cb545e5f892/go.mod h1:S6p41RzIBKoeGAdrziksWkiijnZXql9YcNsc23t0u+8= storj.io/infectious v0.0.2 h1:rGIdDC/6gNYAStsxsZU79D/MqFjNyJc1tsyyj9sTl7Q= storj.io/infectious v0.0.2/go.mod h1:QEjKKww28Sjl1x8iDsjBpOM4r1Yp8RsowNcItsZJ1Vs= -storj.io/picobuf v0.0.2-0.20230906122608-c4ba17033c6c h1:or/DtG5uaZpzimL61ahlgAA+MTYn/U3txz4fe+XBFUg= -storj.io/picobuf v0.0.2-0.20230906122608-c4ba17033c6c/go.mod h1:JCuc3C0gzCJHQ4J6SOx/Yjg+QTpX0D+Fvs5H46FETCk= -storj.io/uplink v1.12.2 h1:lqgOsMhUI2hP5N1twU9zmvhlu6rgyCsKg/MrnzSZGjk= -storj.io/uplink v1.12.2/go.mod h1:FijbSlvLWgzDg/gYyE+rVxTNOlQhCGTn7qPsbFHgeQU= +storj.io/picobuf v0.0.3 h1:xAUPB5ZUGfxkqd3bnw3zp01kkWb9wlhg4vtZWUs2S9A= +storj.io/picobuf v0.0.3/go.mod h1:4V4xelV1RSCck5GgmkL/Txw9l6IfX3XcBzegmL5Kudo= +storj.io/uplink v1.13.0 h1:MAwzMaO4F86n2sMdNm7/m7LVyf8KD0FP+72h1H+HuRE= +storj.io/uplink v1.13.0/go.mod h1:MT8+V7qddgn1ra09piq3Idy4IUva2S90S7me7U8n6cE= diff --git a/k8s/charts/seaweedfs/Chart.yaml b/k8s/charts/seaweedfs/Chart.yaml index e7ed1f472..073679afd 100644 --- a/k8s/charts/seaweedfs/Chart.yaml +++ b/k8s/charts/seaweedfs/Chart.yaml @@ -1,5 +1,6 @@ apiVersion: v1 description: SeaweedFS name: seaweedfs -appVersion: "3.68" -version: 3.68.0 +appVersion: "3.71" +# Dev note: Trigger a helm chart release by `git tag -a helm-<version>` +version: 4.0.0 diff --git a/k8s/charts/seaweedfs/templates/_helpers.tpl b/k8s/charts/seaweedfs/templates/_helpers.tpl index f6f7b8991..d8261eb32 100644 --- a/k8s/charts/seaweedfs/templates/_helpers.tpl +++ b/k8s/charts/seaweedfs/templates/_helpers.tpl @@ -49,25 +49,7 @@ Inject extra environment vars in the format key:value, if populated {{- $imageOverride := .Values.filer.imageOverride -}} {{- printf "%s" $imageOverride -}} {{- else -}} -{{- $registryName := default .Values.image.registry .Values.global.localRegistry | toString -}} -{{- $repositoryName := .Values.image.repository | toString -}} -{{- $name := .Values.global.imageName | toString -}} -{{- $tag := .Chart.AppVersion | toString -}} -{{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}} -{{- end -}} -{{- end -}} - -{{/* Return the proper dbSchema image */}} -{{- define "filer.dbSchema.image" -}} -{{- if .Values.filer.dbSchema.imageOverride -}} -{{- $imageOverride := .Values.filer.dbSchema.imageOverride -}} -{{- printf "%s" $imageOverride -}} -{{- else -}} -{{- $registryName := default .Values.global.registry .Values.global.localRegistry | toString -}} -{{- $repositoryName := .Values.global.repository | toString -}} -{{- $name := .Values.filer.dbSchema.imageName | toString -}} -{{- $tag := .Values.filer.dbSchema.imageTag | toString -}} -{{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}} +{{- include "common.image" . }} {{- end -}} {{- end -}} @@ -77,11 +59,7 @@ Inject extra environment vars in the format key:value, if populated {{- $imageOverride := .Values.master.imageOverride -}} {{- printf "%s" $imageOverride -}} {{- else -}} -{{- $registryName := default .Values.image.registry .Values.global.localRegistry | toString -}} -{{- $repositoryName := .Values.image.repository | toString -}} -{{- $name := .Values.global.imageName | toString -}} -{{- $tag := .Chart.AppVersion | toString -}} -{{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}} +{{- include "common.image" . }} {{- end -}} {{- end -}} @@ -91,11 +69,7 @@ Inject extra environment vars in the format key:value, if populated {{- $imageOverride := .Values.s3.imageOverride -}} {{- printf "%s" $imageOverride -}} {{- else -}} -{{- $registryName := default .Values.image.registry .Values.global.localRegistry | toString -}} -{{- $repositoryName := .Values.image.repository | toString -}} -{{- $name := .Values.global.imageName | toString -}} -{{- $tag := .Chart.AppVersion | toString -}} -{{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}} +{{- include "common.image" . }} {{- end -}} {{- end -}} @@ -105,11 +79,20 @@ Inject extra environment vars in the format key:value, if populated {{- $imageOverride := .Values.volume.imageOverride -}} {{- printf "%s" $imageOverride -}} {{- else -}} -{{- $registryName := default .Values.image.registry .Values.global.localRegistry | toString -}} +{{- include "common.image" . }} +{{- end -}} +{{- end -}} + +{{/* Computes the container image name for all components (if they are not overridden) */}} +{{- define "common.image" -}} +{{- $registryName := default .Values.image.registry .Values.global.registry | toString -}} {{- $repositoryName := .Values.image.repository | toString -}} {{- $name := .Values.global.imageName | toString -}} {{- $tag := .Chart.AppVersion | toString -}} -{{- printf "%s%s%s:%s" $registryName $repositoryName $name $tag -}} +{{- if $registryName -}} +{{- printf "%s/%s%s:%s" $registryName $repositoryName $name $tag -}} +{{- else -}} +{{- printf "%s%s:%s" $repositoryName $name $tag -}} {{- end -}} {{- end -}} diff --git a/k8s/charts/seaweedfs/templates/ca-cert.yaml b/k8s/charts/seaweedfs/templates/ca-cert.yaml index fc002aebe..0fd6615e1 100644 --- a/k8s/charts/seaweedfs/templates/ca-cert.yaml +++ b/k8s/charts/seaweedfs/templates/ca-cert.yaml @@ -14,6 +14,6 @@ spec: commonName: "{{ template "seaweedfs.name" . }}-root-ca" isCA: true issuerRef: - name: {{ template "seaweedfs.name" . }}-clusterissuer - kind: ClusterIssuer + name: {{ template "seaweedfs.name" . }}-issuer + kind: Issuer {{- end }} diff --git a/k8s/charts/seaweedfs/templates/cert-clusterissuer.yaml b/k8s/charts/seaweedfs/templates/cert-issuer.yaml index 42e080fd6..9f243d07c 100644 --- a/k8s/charts/seaweedfs/templates/cert-clusterissuer.yaml +++ b/k8s/charts/seaweedfs/templates/cert-issuer.yaml @@ -1,8 +1,8 @@ {{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}} apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }} -kind: ClusterIssuer +kind: Issuer metadata: - name: {{ template "seaweedfs.name" . }}-clusterissuer + name: {{ template "seaweedfs.name" . }}-issuer labels: app.kubernetes.io/name: {{ template "seaweedfs.name" . }} helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} diff --git a/k8s/charts/seaweedfs/templates/cluster-role.yaml b/k8s/charts/seaweedfs/templates/cluster-role.yaml new file mode 100644 index 000000000..154de0675 --- /dev/null +++ b/k8s/charts/seaweedfs/templates/cluster-role.yaml @@ -0,0 +1,35 @@ +{{- if .Values.global.createClusterRole }} +#hack for delete pod master after migration +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.global.serviceAccountName }}-rw-cr + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:serviceaccount:{{ .Values.global.serviceAccountName }}:default + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +subjects: + - kind: ServiceAccount + name: {{ .Values.global.serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Values.global.serviceAccountName }}-rw-cr +{{- end }}
\ No newline at end of file diff --git a/k8s/charts/seaweedfs/templates/cosi-bucket-class.yaml b/k8s/charts/seaweedfs/templates/cosi-bucket-class.yaml new file mode 100644 index 000000000..e5503abd8 --- /dev/null +++ b/k8s/charts/seaweedfs/templates/cosi-bucket-class.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.cosi.enabled .Values.cosi.bucketClassName }} +--- +kind: BucketClass +apiVersion: objectstorage.k8s.io/v1alpha1 +metadata: + name: {{ .Values.cosi.bucketClassName }} +driverName: {{ .Values.cosi.driverName }} +deletionPolicy: Delete +--- +kind: BucketAccessClass +apiVersion: objectstorage.k8s.io/v1alpha1 +metadata: + name: {{ .Values.cosi.bucketClassName }} +driverName: {{ .Values.cosi.driverName }} +authenticationType: KEY +{{- end }} diff --git a/k8s/charts/seaweedfs/templates/cosi-cluster-role.yaml b/k8s/charts/seaweedfs/templates/cosi-cluster-role.yaml new file mode 100644 index 000000000..75d3ec32c --- /dev/null +++ b/k8s/charts/seaweedfs/templates/cosi-cluster-role.yaml @@ -0,0 +1,69 @@ +{{- if .Values.cosi.enabled }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.global.serviceAccountName }}-objectstorage-provisioner + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +rules: +- apiGroups: ["objectstorage.k8s.io"] + resources: + - "buckets" + - "bucketaccesses" + - "bucketclaims" + - "bucketaccessclasses" + - "buckets/status" + - "bucketaccesses/status" + - "bucketclaims/status" + - "bucketaccessclasses/status" + verbs: + - "get" + - "list" + - "watch" + - "update" + - "create" + - "delete" +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: + - "get" + - "watch" + - "list" + - "delete" + - "update" + - "create" +- apiGroups: [""] + resources: + - "secrets" + - "events" + verbs: + - "get" + - "list" + - "watch" + - "update" + - "create" + - "delete" + - "patch" +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.global.serviceAccountName }}-objectstorage-provisioner + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +subjects: + - kind: ServiceAccount + name: {{ .Values.global.serviceAccountName }}-objectstorage-provisioner + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Values.global.serviceAccountName }}-objectstorage-provisioner + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/k8s/charts/seaweedfs/templates/cosi-deployment.yaml b/k8s/charts/seaweedfs/templates/cosi-deployment.yaml new file mode 100644 index 000000000..15e5fa6d4 --- /dev/null +++ b/k8s/charts/seaweedfs/templates/cosi-deployment.yaml @@ -0,0 +1,204 @@ +{{- if .Values.cosi.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "seaweedfs.name" . }}-objectstorage-provisioner + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + replicas: {{ .Values.cosi.replicas }} + selector: + matchLabels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: objectstorage-provisioner + template: + metadata: + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: objectstorage-provisioner + {{ with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.cosi.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{ with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.cosi.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + restartPolicy: {{ default .Values.global.restartPolicy .Values.cosi.restartPolicy }} + {{- if .Values.cosi.tolerations }} + tolerations: + {{ tpl .Values.cosi.tolerations . | nindent 8 | trim }} + {{- end }} + {{- include "seaweedfs.imagePullSecrets" . | nindent 6 }} + terminationGracePeriodSeconds: 10 + {{- if .Values.cosi.priorityClassName }} + priorityClassName: {{ .Values.cosi.priorityClassName | quote }} + {{- end }} + enableServiceLinks: false + serviceAccountName: {{ .Values.global.serviceAccountName }}-objectstorage-provisioner + {{- if .Values.cosi.initContainers }} + initContainers: + {{ tpl .Values.cosi.initContainers . | nindent 8 | trim }} + {{- end }} + {{- if .Values.cosi.podSecurityContext.enabled }} + securityContext: {{- omit .Values.cosi.podSecurityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + containers: + - name: seaweedfs-cosi-driver + image: "{{ .Values.cosi.image }}" + imagePullPolicy: {{ default "IfNotPresent" .Values.global.imagePullPolicy }} + env: + - name: DRIVERNAME + value: "{{ .Values.cosi.driverName }}" + - name: ENDPOINT + {{- if .Values.cosi.endpoint }} + value: "{{ .Values.cosi.endpoint }}" + {{- else if .Values.s3.ingress.enabled }} + value: "{{ printf "https://%s" .Values.s3.ingress.host }}" + {{- else if .Values.s3.enabled }} + value: "{{ printf "https://%s-s3.%s.svc" (include "seaweedfs.name" .) .Release.Namespace }}" + {{- else }} + value: "{{ printf "https://%s-filer.%s.svc" (include "seaweedfs.name" .) .Release.Namespace }}" + {{- end }} + {{- with .Values.cosi.region }} + - name: REGION + value: "{{ . }}" + {{- end }} + - name: SEAWEEDFS_FILER + value: "{{ template "seaweedfs.name" . }}-filer:{{ .Values.filer.grpcPort }}" + {{- if .Values.global.enableSecurity }} + - name: WEED_GRPC_CLIENT_KEY + value: /usr/local/share/ca-certificates/client/tls.key + - name: WEED_GRPC_CLIENT_CERT + value: /usr/local/share/ca-certificates/client/tls.crt + - name: WEED_GRPC_CA + value: /usr/local/share/ca-certificates/client/ca.crt + {{- end }} + {{- if .Values.cosi.extraEnvironmentVars }} + {{- range $key, $value := .Values.cosi.extraEnvironmentVars }} + - name: {{ $key }} + {{- if kindIs "string" $value }} + value: {{ $value | quote }} + {{- else }} + valueFrom: + {{ toYaml $value | nindent 16 | trim }} + {{- end -}} + {{- end }} + {{- end }} + {{- if .Values.global.extraEnvironmentVars }} + {{- range $key, $value := .Values.global.extraEnvironmentVars }} + - name: {{ $key }} + {{- if kindIs "string" $value }} + value: {{ $value | quote }} + {{- else }} + valueFrom: + {{ toYaml $value | nindent 16 | trim }} + {{- end -}} + {{- end }} + {{- end }} + volumeMounts: + - mountPath: /var/lib/cosi + name: socket + {{- if .Values.cosi.enableAuth }} + - mountPath: /etc/sw + name: config-users + readOnly: true + {{- end }} + {{- if .Values.global.enableSecurity }} + - name: security-config + readOnly: true + mountPath: /etc/seaweedfs/security.toml + subPath: security.toml + - name: ca-cert + readOnly: true + mountPath: /usr/local/share/ca-certificates/ca/ + - name: master-cert + readOnly: true + mountPath: /usr/local/share/ca-certificates/master/ + - name: volume-cert + readOnly: true + mountPath: /usr/local/share/ca-certificates/volume/ + - name: filer-cert + readOnly: true + mountPath: /usr/local/share/ca-certificates/filer/ + - name: client-cert + readOnly: true + mountPath: /usr/local/share/ca-certificates/client/ + {{- end }} + {{ tpl .Values.cosi.extraVolumeMounts . | nindent 12 | trim }} + - name: seaweedfs-cosi-sidecar + image: "{{ .Values.cosi.sidecar.image }}" + imagePullPolicy: {{ default "IfNotPresent" .Values.global.imagePullPolicy }} + args: + - {{ printf "--v=%s" (default "5" .Values.cosi.sidecar.logLevel) }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - mountPath: /var/lib/cosi + name: socket + {{- with .Values.cosi.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.cosi.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.cosi.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + {{- if .Values.cosi.sidecars }} + {{- include "common.tplvalues.render" (dict "value" .Values.cosi.sidecars "context" $) | nindent 8 }} + {{- end }} + volumes: + - name: socket + emptyDir: {} + {{- if .Values.cosi.enableAuth }} + - name: config-users + secret: + defaultMode: 420 + {{- if .Values.cosi.existingConfigSecret }} + secretName: {{ .Values.cosi.existingConfigSecret }} + {{- else }} + secretName: seaweedfs-client-cert + {{- end }} + {{- end }} + {{- if .Values.global.enableSecurity }} + - name: security-config + configMap: + name: {{ template "seaweedfs.name" . }}-security-config + - name: ca-cert + secret: + secretName: {{ template "seaweedfs.name" . }}-ca-cert + - name: master-cert + secret: + secretName: {{ template "seaweedfs.name" . }}-master-cert + - name: volume-cert + secret: + secretName: {{ template "seaweedfs.name" . }}-volume-cert + - name: filer-cert + secret: + secretName: {{ template "seaweedfs.name" . }}-filer-cert + - name: client-cert + secret: + secretName: {{ template "seaweedfs.name" . }}-client-cert + {{- end }} + {{ tpl .Values.cosi.extraVolumes . | indent 8 | trim }} + {{- if .Values.cosi.nodeSelector }} + nodeSelector: + {{ tpl .Values.cosi.nodeSelector . | indent 8 | trim }} + {{- end }} +{{- end }} diff --git a/k8s/charts/seaweedfs/templates/cosi-service-account.yaml b/k8s/charts/seaweedfs/templates/cosi-service-account.yaml new file mode 100644 index 000000000..78227fdeb --- /dev/null +++ b/k8s/charts/seaweedfs/templates/cosi-service-account.yaml @@ -0,0 +1,13 @@ +{{- if .Values.cosi.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.global.serviceAccountName }}-objectstorage-provisioner + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +automountServiceAccountToken: {{ .Values.global.automountServiceAccountToken }} +{{- end }} diff --git a/k8s/charts/seaweedfs/templates/filer-statefulset.yaml b/k8s/charts/seaweedfs/templates/filer-statefulset.yaml index 22d5197c7..49b62e867 100644 --- a/k8s/charts/seaweedfs/templates/filer-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/filer-statefulset.yaml @@ -32,19 +32,25 @@ spec: helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/component: filer - {{ with .Values.podLabels }} + {{- with .Values.podLabels }} {{- toYaml . | nindent 8 }} {{- end }} {{- with .Values.filer.podLabels }} {{- toYaml . | nindent 8 }} {{- end }} annotations: - {{ with .Values.podAnnotations }} + {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} {{- with .Values.filer.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} + {{- if .Values.filer.s3.existingConfigSecret }} + {{- $configSecret := (lookup "v1" "Secret" .Release.Namespace .Values.filer.s3.existingConfigSecret) | default dict }} + checksum/s3config: {{ $configSecret | toYaml | sha256sum }} + {{- else }} + checksum/s3config: {{ include (print .Template.BasePath "/s3-secret.yaml") . | sha256sum }} + {{- end }} spec: restartPolicy: {{ default .Values.global.restartPolicy .Values.filer.restartPolicy }} {{- if .Values.filer.affinity }} @@ -56,9 +62,7 @@ spec: {{ tpl .Values.filer.tolerations . | nindent 8 | trim }} {{- end }} {{- include "seaweedfs.imagePullSecrets" . | nindent 6 }} - {{- if .Values.global.createClusterRole }} serviceAccountName: {{ .Values.filer.serviceAccountName | default .Values.global.serviceAccountName | quote }} # for deleting statefulset pods after migration - {{- end }} terminationGracePeriodSeconds: 60 {{- if .Values.filer.priorityClassName }} priorityClassName: {{ .Values.filer.priorityClassName | quote }} @@ -269,9 +273,9 @@ spec: failureThreshold: {{ .Values.filer.livenessProbe.failureThreshold }} timeoutSeconds: {{ .Values.filer.livenessProbe.timeoutSeconds }} {{- end }} - {{- if .Values.filer.resources }} + {{- with .Values.filer.resources }} resources: - {{ tpl .Values.filer.resources . | nindent 12 | trim }} + {{- toYaml . | nindent 12 }} {{- end }} {{- if .Values.filer.containerSecurityContext.enabled }} securityContext: {{- omit .Values.filer.containerSecurityContext "enabled" | toYaml | nindent 12 }} diff --git a/k8s/charts/seaweedfs/templates/master-statefulset.yaml b/k8s/charts/seaweedfs/templates/master-statefulset.yaml index 358f7f1cf..73d1f9fb3 100644 --- a/k8s/charts/seaweedfs/templates/master-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/master-statefulset.yaml @@ -227,9 +227,9 @@ spec: failureThreshold: {{ .Values.master.livenessProbe.failureThreshold }} timeoutSeconds: {{ .Values.master.livenessProbe.timeoutSeconds }} {{- end }} - {{- if .Values.master.resources }} + {{- with .Values.master.resources }} resources: - {{ tpl .Values.master.resources . | nindent 12 | trim }} + {{- toYaml . | nindent 12 }} {{- end }} {{- if .Values.master.containerSecurityContext.enabled }} securityContext: {{- omit .Values.master.containerSecurityContext "enabled" | toYaml | nindent 12 }} diff --git a/k8s/charts/seaweedfs/templates/s3-deployment.yaml b/k8s/charts/seaweedfs/templates/s3-deployment.yaml index 778cac60f..b678a0eff 100644 --- a/k8s/charts/seaweedfs/templates/s3-deployment.yaml +++ b/k8s/charts/seaweedfs/templates/s3-deployment.yaml @@ -204,9 +204,9 @@ spec: failureThreshold: {{ .Values.s3.livenessProbe.failureThreshold }} timeoutSeconds: {{ .Values.s3.livenessProbe.timeoutSeconds }} {{- end }} - {{- if .Values.s3.resources }} + {{- with .Values.s3.resources }} resources: - {{ tpl .Values.s3.resources . | nindent 12 | trim }} + {{- toYaml . | nindent 12 }} {{- end }} {{- if .Values.s3.containerSecurityContext.enabled }} securityContext: {{- omit .Values.s3.containerSecurityContext "enabled" | toYaml | nindent 12 }} diff --git a/k8s/charts/seaweedfs/templates/service-account.yaml b/k8s/charts/seaweedfs/templates/service-account.yaml index 56f18ac5b..429158a27 100644 --- a/k8s/charts/seaweedfs/templates/service-account.yaml +++ b/k8s/charts/seaweedfs/templates/service-account.yaml @@ -1,20 +1,3 @@ -{{- if .Values.global.createClusterRole }} -#hack for delete pod master after migration ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ .Values.global.serviceAccountName }}-rw-cr - labels: - app.kubernetes.io/name: {{ template "seaweedfs.name" . }} - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Release.Name }} -rules: - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] ---- apiVersion: v1 kind: ServiceAccount metadata: @@ -25,22 +8,4 @@ metadata: helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: system:serviceaccount:{{ .Values.global.serviceAccountName }}:default - labels: - app.kubernetes.io/name: {{ template "seaweedfs.name" . }} - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Release.Name }} -subjects: -- kind: ServiceAccount - name: {{ .Values.global.serviceAccountName }} - namespace: {{ .Release.Namespace }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ .Values.global.serviceAccountName }}-rw-cr -{{- end }} +automountServiceAccountToken: {{ .Values.global.automountServiceAccountToken }}
\ No newline at end of file diff --git a/k8s/charts/seaweedfs/templates/volume-resize-hook.yaml b/k8s/charts/seaweedfs/templates/volume-resize-hook.yaml new file mode 100644 index 000000000..436a785e1 --- /dev/null +++ b/k8s/charts/seaweedfs/templates/volume-resize-hook.yaml @@ -0,0 +1,114 @@ +{{- if and .Values.volume.enabled .Values.volume.resizeHook.enabled }} +{{- $seaweedfsName := include "seaweedfs.name" $ }} +{{- $replicas := int .Values.volume.replicas -}} +{{- $statefulsetName := printf "%s-volume" $seaweedfsName -}} +{{- $statefulset := (lookup "apps/v1" "StatefulSet" .Release.Namespace $statefulsetName) -}} + +{{/* Check for changes in volumeClaimTemplates */}} +{{- $templateChangesRequired := false -}} +{{- if $statefulset -}} + {{- range $dir := .Values.volume.dataDirs -}} + {{- if eq .type "persistentVolumeClaim" -}} + {{- $desiredSize := .size -}} + {{- range $statefulset.spec.volumeClaimTemplates -}} + {{- if and (eq .metadata.name $dir.name) (ne .spec.resources.requests.storage $desiredSize) -}} + {{- $templateChangesRequired = true -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* Check for the need for patching existing PVCs */}} +{{- $pvcChangesRequired := false -}} +{{- range $dir := .Values.volume.dataDirs -}} + {{- if eq .type "persistentVolumeClaim" -}} + {{- $desiredSize := .size -}} + {{- range $i, $e := until $replicas }} + {{- $pvcName := printf "%s-%s-volume-%d" $dir.name $seaweedfsName $e -}} + {{- $currentPVC := (lookup "v1" "PersistentVolumeClaim" $.Release.Namespace $pvcName) -}} + {{- if and $currentPVC (ne ($currentPVC.spec.resources.requests.storage | toString) $desiredSize) -}} + {{- $pvcChangesRequired = true -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{- if or $templateChangesRequired $pvcChangesRequired }} +apiVersion: batch/v1 +kind: Job +metadata: + name: "{{ $seaweedfsName }}-volume-resize-hook" + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation +spec: + template: + spec: + serviceAccountName: {{ $seaweedfsName }}-volume-resize-hook + restartPolicy: Never + backoffLimit: 1 + containers: + - name: resize + image: {{ .Values.volume.resizeHook.image }} + command: ["sh", "-xec"] + args: + - | + {{- if $pvcChangesRequired -}} + {{- range $dir := .Values.volume.dataDirs -}} + {{- if eq .type "persistentVolumeClaim" -}} + {{- $desiredSize := .size -}} + {{- range $i, $e := until $replicas }} + kubectl patch pvc {{ printf "%s-%s-volume-%d" $dir.name $seaweedfsName $e }} -p '{"spec":{"resources":{"requests":{"storage":"{{ $desiredSize }}"}}}}' + {{- end }} + {{- end }} + {{- end }} + {{- end -}} + + {{- if $templateChangesRequired }} + kubectl delete statefulset {{ $statefulsetName }} --cascade=orphan + {{- end }} +{{- end }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $seaweedfsName }}-volume-resize-hook + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-5" + helm.sh/hook-delete-policy: before-hook-creation +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $seaweedfsName }}-volume-resize-hook + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-5" + helm.sh/hook-delete-policy: before-hook-creation +rules: + - apiGroups: ["apps"] + resources: ["statefulsets"] + verbs: ["delete", "get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["patch", "get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $seaweedfsName }}-volume-resize-hook + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-5" + helm.sh/hook-delete-policy: before-hook-creation +subjects: + - kind: ServiceAccount + name: {{ $seaweedfsName }}-volume-resize-hook +roleRef: + kind: Role + name: {{ $seaweedfsName }}-volume-resize-hook + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/k8s/charts/seaweedfs/templates/volume-statefulset.yaml b/k8s/charts/seaweedfs/templates/volume-statefulset.yaml index 5a2c62505..0a70d8c44 100644 --- a/k8s/charts/seaweedfs/templates/volume-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/volume-statefulset.yaml @@ -242,9 +242,9 @@ spec: failureThreshold: {{ .Values.volume.livenessProbe.failureThreshold }} timeoutSeconds: {{ .Values.volume.livenessProbe.timeoutSeconds }} {{- end }} - {{- if .Values.volume.resources }} + {{- with .Values.volume.resources }} resources: - {{ tpl .Values.volume.resources . | nindent 12 | trim }} + {{- toYaml . | nindent 12 }} {{- end }} {{- if .Values.volume.containerSecurityContext.enabled }} securityContext: {{- omit .Values.volume.containerSecurityContext "enabled" | toYaml | nindent 12 }} diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index fc16bd537..e53fa5969 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -19,6 +19,7 @@ global: filerRead: false # we will use this serviceAccountName for all ClusterRoles/ClusterRoleBindings serviceAccountName: "seaweedfs" + automountServiceAccountToken: true certificates: alphacrds: false monitoring: @@ -48,9 +49,6 @@ image: master: enabled: true - repository: null - imageName: null - imageTag: null imageOverride: null restartPolicy: null replicas: 1 @@ -140,7 +138,7 @@ master: # should map directly to the value of the resources field for a PodSpec, # formatted as a multi-line string. By default no direct resource request # is made. - resources: null + resources: {} # updatePartition is used to control a careful rolling update of SeaweedFS # masters. @@ -256,9 +254,6 @@ master: volume: enabled: true - repository: null - imageName: null - imageTag: null imageOverride: null restartPolicy: null port: 8080 @@ -310,6 +305,11 @@ volume: # size: "800Gi" # maxVolumes: 0 + # This will automatically create a job for patching Kubernetes resources if the dataDirs type is 'persistentVolumeClaim' and the size has changed. + resizeHook: + enabled: true + image: bitnami/kubectl + # idx can be defined by: # # idx: @@ -401,7 +401,7 @@ volume: # should map directly to the value of the resources field for a PodSpec, # formatted as a multi-line string. By default no direct resource request # is made. - resources: null + resources: {} # Toleration Settings for server pods # This should be a multi-line string matching the Toleration array @@ -473,9 +473,6 @@ volume: filer: enabled: true - repository: null - imageName: null - imageTag: null imageOverride: null restartPolicy: null replicas: 1 @@ -593,7 +590,7 @@ filer: # should map directly to the value of the resources field for a PodSpec, # formatted as a multi-line string. By default no direct resource request # is made. - resources: null + resources: {} # Toleration Settings for server pods # This should be a multi-line string matching the Toleration array @@ -740,9 +737,7 @@ filer: s3: enabled: false - repository: null - imageName: null - imageTag: null + imageOverride: null restartPolicy: null replicas: 1 bindAddress: 0.0.0.0 @@ -789,7 +784,7 @@ s3: # should map directly to the value of the resources field for a PodSpec, # formatted as a multi-line string. By default no direct resource request # is made. - resources: null + resources: {} # Toleration Settings for server pods # This should be a multi-line string matching the Toleration array @@ -877,10 +872,36 @@ s3: annotations: {} tls: [] +# Deploy Kubernetes COSI Driver for SeaweedFS +# Requires COSI CRDs and controller to be installed in the cluster +# For more information, visit: https://container-object-storage-interface.github.io/docs/deployment-guide +cosi: + enabled: false + image: "ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.1.1" + driverName: "seaweedfs.objectstorage.k8s.io" + bucketClassName: "seaweedfs" + endpoint: "" + region: "" + + sidecar: + image: gcr.io/k8s-staging-sig-storage/objectstorage-sidecar/objectstorage-sidecar:v20230130-v0.1.0-24-gc0cf995 + + # enable user & permission to s3 (need to inject to all services) + enableAuth: false + # set to the name of an existing kubernetes Secret with the s3 json config file + # should have a secret key called seaweedfs_s3_config with an inline json configure + existingConfigSecret: null + + podSecurityContext: {} + containerSecurityContext: {} + + extraVolumes: "" + extraVolumeMounts: "" + certificates: commonName: "SeaweedFS CA" ipAddresses: [] - keyAlgorithm: rsa + keyAlgorithm: RSA keySize: 2048 duration: 2160h # 90d renewBefore: 360h # 15d diff --git a/note/keepsec.png b/note/keepsec.png Binary files differnew file mode 100644 index 000000000..14159dc7b --- /dev/null +++ b/note/keepsec.png diff --git a/note/piknik.png b/note/piknik.png Binary files differindex e8cbc30e6..b4fbcb40b 100644 --- a/note/piknik.png +++ b/note/piknik.png diff --git a/note/seaweedfs.svg b/note/seaweedfs.svg new file mode 100644 index 000000000..61fce5681 --- /dev/null +++ b/note/seaweedfs.svg @@ -0,0 +1,317 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- Created with Inkscape (http://www.inkscape.org/) --> + +<svg + version="1.1" + id="svg824" + width="437" + height="436.59174" + viewBox="0 0 437.00001 436.59175" + sodipodi:docname="seaweedfs.svg.2024_07_29_16_04_23.0.svg" + inkscape:version="1.1.1 (c3084ef, 2021-09-22)" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns="http://www.w3.org/2000/svg" + xmlns:svg="http://www.w3.org/2000/svg"> + <defs + id="defs828"> + <linearGradient + inkscape:collect="always" + id="linearGradient25431"> + <stop + style="stop-color:#0569b4;stop-opacity:1;" + offset="0" + id="stop25427" /> + <stop + style="stop-color:#4184c8;stop-opacity:1" + offset="0.4744136" + id="stop25981" /> + <stop + style="stop-color:#0569b4;stop-opacity:0.92477876" + offset="1" + id="stop25429" /> + </linearGradient> + <linearGradient + inkscape:collect="always" + id="linearGradient22881"> + <stop + style="stop-color:#034f87;stop-opacity:1" + offset="0" + id="stop22877" /> + <stop + style="stop-color:#50b1fa;stop-opacity:0" + offset="1" + id="stop22879" /> + </linearGradient> + <linearGradient + inkscape:collect="always" + id="linearGradient21639"> + <stop + style="stop-color:#2e82cf;stop-opacity:1" + offset="0" + id="stop21635" /> + <stop + style="stop-color:#0066a5;stop-opacity:1" + offset="1" + id="stop21637" /> + </linearGradient> + <linearGradient + inkscape:collect="always" + id="linearGradient19625"> + <stop + style="stop-color:#88a3d0;stop-opacity:1" + offset="0" + id="stop19621" /> + <stop + style="stop-color:#5d83bf;stop-opacity:1" + offset="1" + id="stop19623" /> + </linearGradient> + <radialGradient + inkscape:collect="always" + xlink:href="#linearGradient19625" + id="radialGradient19627" + cx="25.020868" + cy="23.369808" + fx="25.020868" + fy="23.369808" + r="3.9694827" + gradientTransform="matrix(2.4252415,-0.02082447,0.02181189,2.5402371,-133.80669,-136.47206)" + gradientUnits="userSpaceOnUse" /> + <radialGradient + inkscape:collect="always" + xlink:href="#linearGradient19625" + id="radialGradient21589" + cx="10.763105" + cy="45.500729" + fx="10.763105" + fy="45.500729" + r="2.3373196" + gradientTransform="matrix(3.1669355,0,0,3.2457102,-151.85478,-155.22323)" + gradientUnits="userSpaceOnUse" /> + <radialGradient + inkscape:collect="always" + xlink:href="#linearGradient19625" + id="radialGradient21591" + cx="25.151409" + cy="106.26995" + fx="25.151409" + fy="106.26995" + r="2.347388" + gradientTransform="matrix(3.1669355,0,0,3.2908805,-151.85478,-164.81056)" + gradientUnits="userSpaceOnUse" /> + <radialGradient + inkscape:collect="always" + xlink:href="#linearGradient19625" + id="radialGradient21593" + cx="21.516373" + cy="113.90106" + fx="21.516373" + fy="113.90106" + r="1.2429004" + gradientTransform="matrix(3.1669355,0,0,3.6313007,-151.85478,-204.53064)" + gradientUnits="userSpaceOnUse" /> + <linearGradient + inkscape:collect="always" + xlink:href="#linearGradient21639" + id="linearGradient21641" + x1="114.68082" + y1="98.406364" + x2="109.24162" + y2="136.83397" + gradientUnits="userSpaceOnUse" + gradientTransform="matrix(3.1669355,0,0,3.1669355,-151.85478,-151.63893)" /> + <radialGradient + inkscape:collect="always" + xlink:href="#linearGradient22881" + id="radialGradient22883" + cx="38.979542" + cy="115.14806" + fx="38.979542" + fy="115.14806" + r="27.733946" + gradientTransform="matrix(5.4446674,-0.00544507,0.00556237,5.561902,-241.28022,-421.53182)" + gradientUnits="userSpaceOnUse" /> + <linearGradient + inkscape:collect="always" + xlink:href="#linearGradient25431" + id="linearGradient25433" + x1="80.262825" + y1="93.085884" + x2="80.429276" + y2="32.328072" + gradientUnits="userSpaceOnUse" + gradientTransform="matrix(3.1669355,0,0,3.1669355,-151.85478,-151.63893)" /> + </defs> + <sodipodi:namedview + id="namedview826" + pagecolor="#ffffff" + bordercolor="#666666" + borderopacity="1.0" + inkscape:pageshadow="2" + inkscape:pageopacity="0.0" + inkscape:pagecheckerboard="0" + showgrid="false" + showguides="false" + inkscape:zoom="2.4803992" + inkscape:cx="340.06622" + inkscape:cy="223.95588" + inkscape:window-width="3440" + inkscape:window-height="1387" + inkscape:window-x="0" + inkscape:window-y="25" + inkscape:window-maximized="1" + inkscape:current-layer="g830" + fit-margin-top="0" + fit-margin-left="0" + fit-margin-right="0" + fit-margin-bottom="0" /> + <g + inkscape:groupmode="layer" + inkscape:label="Image" + id="g830" + transform="translate(148.42187,148.31736)"> + <path + style="color:#000000;fill:#005dad;stroke-width:4.7504;-inkscape-stroke:none" + d="m -123.15443,-146.45374 c -12.68161,0 -22.89224,10.21063 -22.89224,22.89225 v 386.05686 c 0,12.68161 10.21063,22.89222 22.89224,22.89222 h 386.47127 c 12.68162,0 22.88609,-10.21061 22.88609,-22.89222 v -386.05686 c 0,-12.68162 -10.20447,-22.89225 -22.88609,-22.89225 z" + id="path28823" /> + <path + id="path11312" + style="opacity:1;fill:url(#linearGradient21641);stroke-width:2.91212" + d="m 270.2248,217.48749 a 63.442303,63.416565 0 0 1 -63.4423,63.41656 63.442303,63.416565 0 0 1 -63.4423,-63.41656 63.442303,63.416565 0 0 1 63.4423,-63.41657 63.442303,63.416565 0 0 1 63.4423,63.41657 z" /> + <path + id="rect27092" + style="opacity:1;fill:#0f5e9c;stroke-width:3.16694" + d="m 184.9966,232.79112 h 40.11811 v 32.09449 H 184.9966 Z" /> + <g + id="g4961" + transform="matrix(3.1669355,0,0,3.1669355,-151.85478,-151.63893)"> + <path + id="path4810" + style="fill:#59686f" + d="m 114.84587,108.23793 a 3.3700454,3.039649 0 0 1 -3.37005,3.03965 3.3700454,3.039649 0 0 1 -3.37004,-3.03965 3.3700454,3.039649 0 0 1 3.37004,-3.03965 3.3700454,3.039649 0 0 1 3.37005,3.03965 z" /> + <path + id="path4810-8" + style="fill:#59686f" + d="m 123.23793,109.95599 a 3.3700454,3.039649 0 0 1 -3.37004,3.03965 3.3700454,3.039649 0 0 1 -3.37005,-3.03965 3.3700454,3.039649 0 0 1 3.37005,-3.03965 3.3700454,3.039649 0 0 1 3.37004,3.03965 z" /> + <path + id="path4810-8-5" + style="fill:#59686f" + d="m 117.75335,114.58154 a 3.3700454,3.039649 0 0 1 -3.37004,3.03965 3.3700454,3.039649 0 0 1 -3.37005,-3.03965 3.3700454,3.039649 0 0 1 3.37005,-3.03965 3.3700454,3.039649 0 0 1 3.37004,3.03965 z" /> + </g> + <path + style="fill:#d3d6da;fill-opacity:1;stroke-width:1.01342" + d="m 199.94434,220.03836 c -4.72833,-1.44048 1.19939,-3.68363 0.36331,-4.00447 -2.33184,-0.89481 0.90695,-3.72783 3.94363,-6.11649 2.66761,-2.09832 2.76312,-2.10424 3.46219,-0.2144 0.39536,1.06884 1.44108,2.2205 2.3238,2.55923 1.87457,0.71931 6.87119,-0.14885 6.87119,-1.45581 0,-1.83815 1.39489,-0.40749 2.53499,2.0948 1.99444,4.37725 1.93275,8.84149 -1.03378,8.84149 -1.77615,0 -3.42808,0.65723 -3.78626,2.02683 -15.46539,4.36873 -11.18204,0.41224 -14.67907,-3.73118 z m 23.40743,-11.5775 c -4.46323,-1.76797 -7.65588,-9.66818 -7.65588,-14.26755 0,-5.96907 5.99371,-12.66774 11.33462,-12.66774 12.01751,0 19.14137,9.51829 13.59078,28.00236 -1.36334,4.54002 -14.74592,-0.0675 -17.26952,-1.06707 z m 8.33334,-7.23106 c 5.04468,-3.53345 -0.0253,-11.70762 -5.66694,-9.13778 -4.5146,2.05699 -2.59116,10.53763 2.37333,10.68454 0.57271,0.0158 2.05483,-0.67912 3.29361,-1.54676 z m -4.58825,-3.21653 c -1.70819,-0.59586 -1.61368,-3.97669 0.12984,-4.64577 1.56162,-0.59921 3.67048,1.09392 3.67048,2.9469 0,1.42683 -2.00749,2.32421 -3.80032,1.69887 z m -36.13914,7.21795 c -7.08963,-3.93735 -6.11038,-17.95959 0.29674,-24.09787 5.94944,-5.69995 19.12316,-5.59693 22.08662,1.49564 1.95837,4.68707 1.3958,9.43307 -1.1006,13.87482 -3.61972,6.44041 -15.49876,11.93957 -21.28263,8.72741 z m 12.75068,-10.18157 c 2.29964,-2.08112 2.21746,-4.61061 0.0377,-7.01938 -2.39183,-2.64297 -5.65415,-2.74757 -7.62782,0.26475 -1.53777,2.34695 -1.81487,3.5515 -0.2771,5.89845 1.91435,2.92166 5.19114,3.27835 7.86739,0.85634 z m -6.57154,-3.76891 c -0.74034,-1.92923 1.83786,-4.08585 3.92642,-3.28439 0.88437,0.33937 1.45559,1.41647 1.26934,2.39357 -0.42465,2.228 -4.4198,2.91298 -5.19576,0.89082 z" + id="path3004" + sodipodi:nodetypes="csssssssccssssssscsssssssssssssssssssss" /> + <path + id="rect7701" + style="fill:#98c6d8;fill-opacity:1;stroke:none;stroke-width:3.16694" + d="m 209.42982,217.34 -1.55721,-0.68517 c -1.30918,-0.28534 -2.14706,0.9841 -2.43237,2.29331 l -0.767,3.50713 c -0.28534,1.30918 0.54025,2.5909 1.84946,2.87621 l 2.85765,0.62474 c 1.30918,0.28534 2.59708,-0.54025 2.88242,-1.84946 l 0.84098,-5.12065 c 0.28534,-1.30918 -0.62043,-0.98356 -1.92961,-1.26886 l -1.24948,-0.27204 c -0.16437,0.83521 -0.33614,1.71309 -0.47007,2.3752 -0.10039,0.49657 -0.19477,0.94701 -0.27838,1.34842 -0.0836,0.4014 -0.15549,0.75575 -0.22263,1.0639 -0.0675,0.30814 -0.12826,0.56878 -0.17925,0.78552 -0.0513,0.21662 -0.0947,0.38612 -0.12984,0.51339 -0.019,0.0637 -0.0285,0.11908 -0.0434,0.16088 -0.0127,0.0415 -0.0285,0.0728 -0.037,0.0928 -0.006,0.009 -0.01,0.0127 -0.0127,0.019 -0.003,0.003 -0.01,0.006 -0.0127,0.006 l -0.006,-0.006 c -0.002,-0.006 -0.006,-0.019 -0.006,-0.0317 -0.002,-0.0222 -0.002,-0.0551 0,-0.0988 0.002,-0.044 8.9e-4,-0.096 0.006,-0.16088 0.01,-0.12953 0.0317,-0.30181 0.0557,-0.51336 0.0253,-0.21155 0.0592,-0.4632 0.0988,-0.75465 0.0396,-0.29136 0.0884,-0.62636 0.1422,-0.99584 0.0538,-0.36949 0.11179,-0.77293 0.17925,-1.21851 0.0678,-0.44562 0.14156,-0.93374 0.22263,-1.45359 0.0567,-0.36224 0.13491,-0.83959 0.19794,-1.23707 z" + sodipodi:nodetypes="ccccccccccsscccccccccsccsscccc" /> + <path + style="fill:#98c6d8;fill-opacity:1;stroke-width:1.01342" + d="m 164.68687,203.38819 c -0.3053,-0.79531 0.12509,-2.01027 0.95586,-2.69984 1.21623,-1.0094 1.82057,-0.94403 3.09992,0.33535 0.87407,0.87404 1.2803,2.08897 0.90277,2.69984 -0.98533,1.59427 -4.30422,1.36979 -4.95855,-0.33535 z" + id="path3004-0" + sodipodi:nodetypes="sssss" /> + <path + style="fill:#98c6d8;fill-opacity:1;stroke-width:1.01342" + d="m 172.07664,191.10213 c -1.07704,-1.29775 -1.0105,-1.86431 0.38263,-3.25742 1.57295,-1.57295 1.7999,-1.57295 3.37285,0 1.39313,1.39311 1.45967,1.95967 0.38263,3.25742 -0.71709,0.86403 -1.64817,1.57099 -2.06905,1.57099 -0.42089,0 -1.35197,-0.70696 -2.06906,-1.57099 z" + id="path3004-0-3" + sodipodi:nodetypes="ssssss" /> + <path + style="fill:#98c6d8;fill-opacity:1;stroke-width:1.01342" + d="m 172.07661,175.90084 c -1.07125,-1.29078 -1.03508,-1.83971 0.20268,-3.0775 1.635,-1.635 4.9066,-0.96338 4.9066,1.00727 0,1.30541 -2.00071,3.64122 -3.11886,3.64122 -0.37766,0 -1.27336,-0.70696 -1.99045,-1.57099 z" + id="path3004-0-3-0" + sodipodi:nodetypes="ssssss" /> + <path + style="fill:#7ba9b9;fill-opacity:1;stroke-width:1.01342" + d="m 207.90174,264.03919 c -10.15221,-1.29227 -23.65127,-4.9897 -25.68761,-6.81645 -3.86933,-3.47109 -1.40755,-19.22406 3.17507,-23.25091 2.76733,-2.4317 13.74666,-5.64136 20.18934,-4.12247 7.39524,1.7435 20.29357,10.45992 21.24979,13.04483 1.12825,3.04998 -0.60881,15.13149 -2.78747,18.12194 -1.89136,2.59603 -9.01205,3.93029 -16.13912,3.02306 z m 7.7742,-7.38783 c 2.2737,-2.05762 2.41105,-6.74104 0.26064,-8.8913 -3.41953,-3.41956 -10.56851,-0.77273 -10.56851,3.91285 0,5.63917 6.24488,8.65533 10.30774,4.97845 z m -5.88759,-2.88318 c -0.81457,-2.12276 0.29484,-4.56038 2.07545,-4.56038 2.18826,0 2.87197,1.69893 1.62296,4.03274 -1.29635,2.42229 -2.88441,2.64886 -3.69841,0.52764 z m -6.49329,0.0557 c 2.0206,-2.43468 0.86919,-5.91583 -2.22905,-6.73921 -3.30505,-0.87831 -4.89684,0.84529 -2.43987,2.64186 2.44573,1.78834 2.05113,2.70099 -1.00972,2.33539 -3.62642,-0.43314 -5.11067,-2.98129 -3.84497,-6.60106 1.28181,-3.66598 5.83919,-4.50395 7.51058,-1.38097 1.32365,2.47328 2.62881,2.67935 3.44806,0.54445 0.99097,-2.58244 -2.96397,-5.55081 -7.39568,-5.55081 -7.35534,0 -10.89245,8.52821 -5.69342,13.72721 2.86981,2.86984 9.62897,3.46323 11.65407,1.02314 z" + id="path3004-5" + sodipodi:nodetypes="sssssssscsssssssssssssssss" /> + <path + style="fill:#a6b3c2;fill-opacity:1;stroke-width:1.01342" + d="m 214.4047,270.20505 c -4.11097,-3.89621 5.40282,-7.50662 10.48578,-0.92975 2.59268,3.12396 -2.266,5.08864 -10.48578,0.92975 z m -42.08316,-8.82203 c -4.11714,-4.11715 2.53361,-5.2733 7.39796,-1.28603 l 4.08662,2.07352 -5.65742,0.42487 c -2.53608,0.003 -5.1583,-0.54221 -5.82716,-1.21107 z m 57.85133,-23.42611 c -0.84624,-1.36923 8.92756,-4.80731 9.99554,-4.14726 0.47463,0.29326 0.86296,2.5586 0.86296,5.03394 0,3.24297 -0.40806,4.50056 -1.4603,4.50056 -0.8032,0 -9.06083,-4.84132 -9.3982,-5.38724 z m -20.20866,-20.35709 c -11.33592,-1.0253 -9.81066,-5.31334 -6.85601,-7.80274 1.70502,-1.43654 2.94361,-2.16441 4.66543,-2.50056 0,0 -0.12313,4.14568 2.26364,4.96571 1.89889,0.6524 4.84809,-0.76551 5.42731,-1.93711 5.16278,-10.44278 2.83877,0.0738 3.97887,2.5761 4.33937,13.45587 -4.83062,6.60927 -9.47924,4.6986 z m -41.27438,0.40593 c -2.0157,-2.23113 -3.66491,-4.51133 -3.66491,-5.06709 0,-1.28724 3.16554,-1.298 6.52126,-0.0222 2.19611,0.83496 13.48149,1.41204 13.48149,4.85782 0,2.24333 -4.37905,7.84561 -4.87157,7.84561 -0.49253,0 -9.45058,-5.38306 -11.46627,-7.61417 z" + id="path3004-2" + sodipodi:nodetypes="ccccsccccssssscssssccsssssss" /> + <path + id="path11158" + style="fill:#4c9cbb;fill-opacity:1;stroke:none;stroke-width:3.16694" + d="m 209.42982,217.34 c -0.063,0.39748 -0.14124,0.87483 -0.19793,1.23707 -0.0814,0.51985 -0.15486,1.00797 -0.22264,1.45359 -0.0678,0.44559 -0.12541,0.84902 -0.17925,1.21851 -0.0538,0.36949 -0.1026,0.70442 -0.14219,0.99584 -0.0396,0.29136 -0.0738,0.54313 -0.0988,0.75465 -0.0253,0.21155 -0.045,0.38371 -0.0557,0.51336 -0.006,0.0649 -0.003,0.11686 -0.006,0.16088 -0.002,0.044 -0.002,0.076 0,0.0988 0.001,0.0127 0.003,0.0253 0.006,0.0317 0,0 0.006,0.006 0.006,0.006 0.003,8.6e-4 0.01,-0.002 0.0127,-0.006 0.003,-0.003 0.006,-0.01 0.0127,-0.019 0.01,-0.019 0.0222,-0.0513 0.037,-0.0928 0.0127,-0.0415 0.0253,-0.0972 0.0434,-0.16088 0.0351,-0.12731 0.0789,-0.29674 0.12985,-0.51339 0.0513,-0.21662 0.11211,-0.47742 0.17924,-0.78553 0.0671,-0.30814 0.13903,-0.66249 0.22264,-1.0639 0.0836,-0.40141 0.17798,-0.85184 0.27837,-1.34842 0.13397,-0.66211 0.30593,-1.53998 0.47007,-2.3752 z" /> + <path + style="fill:url(#radialGradient22883);fill-opacity:1;stroke-width:1.01342" + d="m -53.520044,282.47205 c -13.677994,-5.28318 -28.657188,-17.64053 -36.697077,-30.27388 -10.181919,-15.9992 -13.188469,-27.61076 -11.209229,-43.29121 1.854865,-14.69512 6.891309,-22.9892 21.524343,-35.44643 20.499051,-17.45106 36.862277,-25.35271 55.231355,-26.67071 27.7383838,-1.99027 43.601741,6.8227 51.264012,28.47998 2.578614,7.28841 3.123681,25.71985 1.226338,41.46921 -0.671482,5.57381 -1.730293,15.77375 -2.352928,22.66649 -2.179875,24.13217 -8.589467,36.28259 -22.9880787,43.57763 -3.990551,2.0218 -5.7284955,2.15713 -26.8556133,2.0911 -21.577471,-0.0675 -22.882442,-0.184 -29.143122,-2.60218 z M -146.47366,169.1083 c 0.006,-3.34428 0.19488,-4.59022 0.42043,-2.76876 0.22555,1.82147 0.2209,4.5577 -0.0103,6.08052 -0.23124,1.52282 -0.41578,0.0326 -0.41009,-3.31176 z m -0.23402,-8.1918 c 0.0486,-1.18059 0.28877,-1.42072 0.61228,-0.61228 0.29274,0.73156 0.25674,1.60564 -0.08,1.94239 -0.33675,0.33675 -0.57627,-0.26181 -0.53227,-1.33011 z m 0.08,-81.495812 c 0,-1.393452 0.23002,-1.9635 0.51115,-1.266774 0.28114,0.696725 0.28114,1.836822 0,2.533548 -0.28113,0.696726 -0.51115,0.126677 -0.51115,-1.266774 z" + id="path2364" + sodipodi:nodetypes="ssssssssssssssssssssssss" /> + <path + style="fill:url(#linearGradient25433);fill-opacity:1;stroke-width:1.01342" + d="m 98.4598,156.50835 c -3.065593,-1.0175 -10.454554,-4.18525 -16.419927,-7.03944 -11.150137,-5.33489 -16.662175,-6.4188 -19.288075,-3.7929 -0.768083,0.76808 -4.334964,2.93405 -7.92639,4.81328 -5.522841,2.88983 -7.786211,3.47058 -14.676409,3.76577 -7.17879,0.30754 -9.104996,-0.0241 -16.214709,-2.7916 -9.708498,-3.77914 -16.2585721,-7.46409 -15.9466733,-8.97131 0.125854,-0.60823 -2.688611,-2.73743 -6.2544062,-4.73153 -7.0866516,-3.96308 -9.424002,-6.98175 -12.2155955,-15.77636 -2.12584,-6.69722 -5.501125,-10.56504 -11.078457,-12.69504 -2.488786,-0.95048 -5.105214,-2.98313 -6.545495,-5.08509 -1.319665,-1.92592 -2.991895,-3.50167 -3.716066,-3.50167 -0.72418,0 -2.274284,-2.559404 -3.444673,-5.687567 -1.654549,-4.422166 -2.079242,-7.635101 -1.908857,-14.441225 0.12503,-4.99485 -0.269475,-9.650945 -0.918757,-10.843293 -3.153039,-5.790305 -5.794143,-26.792822 -3.369233,-26.792822 0.517867,0 1.456943,-2.622223 2.086852,-5.827162 2.324157,-11.825305 3.612701,-14.217431 12.903717,-23.955308 9.006096,-9.4392504 15.903774,-19.9040119 13.620964,-20.6649474 -0.654494,-0.2181701 -0.899489,-0.8666984 -0.544447,-1.4411741 0.355052,-0.5744758 1.02185,-6.3500255 1.481781,-12.8345415 0.809418,-11.411973 2.536082,-18.315883 7.510907,-30.031577 1.9474468,-4.586258 6.3060811,-10.721027 7.3903005,-10.401828 0.3954141,0.116416 1.9697199,-0.624086 3.4984535,-1.645553 3.662833,-2.447417 6.410799,-1.663645 12.907618,3.681496 12.910843,10.622162 21.964405,14.065903 31.843695,12.112515 3.781768,-0.747739 5.365457,-0.70768 5.910949,0.149543 0.55638,0.87434 0.74905,0.865207 0.760644,-0.03607 0.0082,-0.663827 -0.491508,-1.520319 -1.111205,-1.903322 -0.647201,-0.399987 0.970147,-1.725986 3.800323,-3.11571 2.709874,-1.330651 7.456638,-5.150517 10.548368,-8.48859 6.037133,-6.518162 12.501662,-9.655531 22.28189,-10.813853 5.634682,-0.667346 15.349483,2.085477 14.565903,4.127448 -0.5838,1.521364 7.18014,5.941057 15.41115,8.772943 4.75022,1.634313 8.49166,1.984687 21.07174,1.973305 16.42013,-0.01488 20.25805,-0.517924 19.47711,-2.553035 -0.63554,-1.656199 2.48487,-2.470694 9.50004,-2.479707 6.35313,-0.0082 8.79382,1.306215 14.70548,7.919233 5.02197,5.617827 10.52056,9.379598 18.76035,12.834608 3.61034,1.513845 6.92894,3.387417 7.37466,4.163494 0.54446,0.947981 2.70624,1.40793 6.58722,1.401546 10.98959,-0.01805 21.05173,-8.38044 28.73069,-23.877224 3.76565,-7.599385 4.14711,-9.121199 4.59292,-18.322784 0.40299,-8.317661 0.14694,-10.92796 -1.49625,-15.252449 -1.38658,-3.64919 -1.66211,-5.537741 -0.91667,-6.28319 0.74543,-0.745449 0.60146,-1.606049 -0.47903,-2.863229 -0.84966,-0.98858 -1.83049,-2.60234 -2.17962,-3.58612 -0.96021,-2.70573 -5.01848,-3.78615 -14.18049,-3.77521 -6.57928,0.008 -9.35861,0.50529 -13.43883,2.40521 -6.5051,3.02907 -8.79727,7.32717 -9.36345,17.557872 -0.33766,6.100395 -0.0545,7.871529 1.71771,10.738787 1.16714,1.88845 1.85769,3.697903 1.5346,4.021001 -0.32309,0.3231 0.82306,1.316853 2.54698,2.208335 3.83022,1.980675 3.94549,1.272649 1.1076,-6.801684 -3.7283,-10.607682 -2.02747,-20.773731 3.81638,-22.810911 1.9756,-0.6887 5.77557,-0.7442 10.98135,-0.1604 8.85041,0.99253 9.50325,1.47998 14.0789,10.512343 3.23445,6.384808 3.19132,7.676734 -0.50991,15.274218 -2.33137,4.785566 -3.94192,6.713216 -6.84058,8.187441 -2.08521,1.060562 -4.01932,2.203516 -4.29801,2.539952 -0.27869,0.336436 -3.99613,0.921435 -8.26101,1.300001 -7.33462,0.651046 -7.85571,0.55955 -9.62749,-1.690503 -1.03023,-1.308341 -3.15353,-3.21711 -4.71848,-4.241708 -4.41483,-2.890567 -6.78218,-8.53117 -6.78218,-16.159821 0,-6.106792 0.42431,-7.964729 3.37608,-14.782643 1.54132,-3.56005 3.82822,-6.13191 4.87651,-5.48405 0.48828,0.30178 2.05116,-0.36646 3.47309,-1.48496 5.85436,-4.60503 21.85416,-4.95117 28.47059,-0.61592 1.33353,0.87376 2.83124,1.33734 3.32822,1.03017 1.38937,-0.85866 4.79902,4.07056 7.22109,10.439196 2.52557,6.640874 3.5642,23.885677 1.88883,31.361481 -1.51807,6.773949 -6.30749,17.281964 -8.09187,17.753632 -3.08115,0.814444 -5.24112,11.420314 -4.86122,23.869319 l 0.34875,11.426516 5.86617,5.5455508 c 3.22641,3.0500597 6.25116,5.3076319 6.72167,5.0168312 1.92641,-1.1905746 6.83792,14.821511 6.97907,22.752563 0.0253,1.393452 0.94621,5.745135 2.04755,9.670402 1.10136,3.925277 2.3277,9.739771 2.72515,12.921097 0.61818,4.947605 0.48593,5.784243 -0.9142,5.784243 -1.26998,0 -1.6369,1.046235 -1.6369,4.667464 0,10.123686 -5.67008,25.202394 -13.71767,36.480108 -2.54954,3.572832 -4.38314,6.904375 -4.07474,7.403424 0.30846,0.499049 -0.80652,3.506703 -2.47772,6.683673 -1.67119,3.17697 -3.03851,6.62497 -3.03851,7.66222 0,4.35584 -4.27239,13.07131 -8.10505,16.53387 -5.19317,4.69171 -11.77298,8.03485 -16.33448,8.29937 -3.18043,0.18445 -15.38709,5.48319 -23.19118,10.06698 -3.12143,1.83341 -11.41725,1.88169 -20.52174,0.11943 -3.76232,-0.72822 -6.84059,-0.93948 -6.84059,-0.46945 0,1.59651 -20.10972,0.98868 -23.38873,-0.70697 -3.33378,-1.72395 -3.43455,-1.68754 -10.33188,3.73391 -6.00021,4.7163 -17.27373,10.5387 -17.73367,9.15885 -0.22152,-0.66461 -1.47462,-0.69861 -3.57258,-0.0969 -4.84598,1.38979 -10.70516,1.03246 -16.92591,-1.03228 z M 236.49547,-72.291215 c 2.11367,-2.063952 3.84304,-4.313094 3.84304,-4.998089 0,-0.684999 1.36812,-2.219635 3.04026,-3.410305 4.08113,-2.906012 3.94153,-6.289385 -0.53363,-12.934623 l -3.57389,-5.306926 h -5.87694 c -4.85118,0 -6.26578,0.413887 -8.10463,2.371272 -4.59858,4.894974 -4.87382,12.241029 -0.52065,13.896108 2.14158,0.814219 2.39503,1.538851 2.39503,6.847278 0,3.265184 0.30402,6.240722 0.6756,6.612311 1.62955,1.629557 5.0988,0.396285 8.65581,-3.077026 z" + id="path2364-8" + sodipodi:nodetypes="ssssssssssssssssssssssssssscsssssssscsssssssssssssssssssssssssssssssssssscsscsssssssssssssssscsssssscssssss" /> + <path + style="fill:#3060ad;fill-opacity:1;stroke:none;stroke-width:3.16694px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" + d="m -40.550031,191.89424 7.776705,-4.36923 4.364243,31.17316 -0.311722,22.44468 -11.657837,5.78725 -7.357789,-13.26882 z" + id="path1136" /> + <path + style="fill:#606368;fill-opacity:1;stroke:none;stroke-width:3.16694px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" + d="m -30.279459,204.35851 1.870389,23.69162 11.845798,4.05251 0.311722,-50.1888 -9.663679,2.49386 z" + id="path7328" /> + <path + style="fill:#eec23b;stroke-width:1.01342" + d="m -40.378652,246.9301 c -7.027373,-3.36863 -16.955266,-13.35636 -20.166089,-20.28761 -1.89061,-4.08132 -3.073726,-5.38952 -5.320451,-5.88299 -3.320434,-0.72928 -3.258733,-0.52511 -1.715371,-5.67638 0.81713,-2.72734 1.747911,-3.91792 3.063009,-3.91792 1.039047,0 5.468766,-3.3476 9.843821,-7.4391 4.375055,-4.09149 9.342684,-8.42746 11.039177,-9.63546 l 3.084525,-2.1964 1.093863,5.10051 c 4.111506,19.17127 4.059527,30.13792 -0.171014,36.08375 -3.209914,4.51142 -2.339245,6.46245 2.883928,6.46245 2.458739,0 3.737237,-0.62503 4.664092,-2.2802 1.636156,-2.92181 1.506204,-20.04049 -0.255794,-33.69619 -2.106949,-16.3292 -2.2028,-15.80972 3.344082,-18.12345 2.640312,-1.10131 4.991261,-1.80646 5.224347,-1.56697 0.233087,0.23942 -0.229729,4.08376 -1.028478,8.54281 -2.846169,15.88886 -1.869901,31.62574 2.169965,34.97855 1.232217,1.02266 1.347037,-1.15489 0.811718,-15.39489 -0.690503,-18.36807 -0.0837,-24.9913 2.592317,-28.29606 1.540093,-1.90194 2.421464,-2.11732 6.278326,-1.53422 7.3978283,1.11837 7.3797578,1.07679 5.1531234,11.85817 -1.0831932,5.24483 -2.2898214,11.93552 -2.6813934,14.8682 l -0.711959,5.33213 2.5957312,-0.98691 c 1.4276546,-0.54278 4.4996042,-1.95815 6.8265449,-3.14527 l 4.2308041,-2.15839 -0.7014192,2.37352 c -0.3857771,1.30547 -0.96833171,6.10465 -1.2945514,10.66488 l -0.59312268,8.29132 -4.96610002,-1.95745 c -2.7313426,-1.0766 -5.4052399,-1.95745 -5.9419849,-1.95745 -0.577396,0 -0.975904,5.58663 -0.975904,13.68116 0,11.02166 -0.274542,13.68116 -1.412402,13.68116 -9.977675,-1.44276 -16.698627,-0.9522 -26.96334,-5.7853 z" + id="path2378" + sodipodi:nodetypes="csssssscsssssssssssssscsscsscssscc" /> + <path + style="fill:#ffffff;stroke-width:1.01342" + d="m -22.959648,251.64273 c 0,-0.40521 0.531712,-5.72164 1.519144,-12.847 0.987447,-7.12538 2.069779,-15.5836 2.405179,-18.79604 0.335392,-3.21248 0.985054,-5.60893 1.443686,-5.32549 2.088984,1.29107 4.041447,27.81982 3.300707,33.96827 -0.470407,3.90457 -1.165971,3.95784 -2.241012,3.77866 -1.929373,-0.32157 -4.184219,-0.49199 -6.427704,-0.77831 z m -25.755436,-29.11136 c -2.752545,-0.4562 -5.32937,-3.88751 -5.32937,-7.0966 0,-2.90889 4.250595,-7.32908 7.047876,-7.32908 4.503667,0 8.029068,7.74519 5.350997,11.75598 -1.192827,1.78641 -4.610865,3.07719 -7.069503,2.6697 z" + id="path2384" + sodipodi:nodetypes="sssssscssssss" /> + <path + style="fill:#0996d1;stroke-width:1.01342" + d="m 196.07647,266.9123 c -20.1534,-5.28261 -23.29525,-10.20207 -22.28427,-34.89238 0.32132,-7.8471 0.83113,-16.09164 1.13297,-18.32117 1.68206,-12.42585 4.07841,-22.92095 6.31427,-27.65444 1.97721,-4.18586 2.2724,-5.67131 1.34829,-6.78478 -3.26007,-3.92817 2.51322,-10.06167 7.13998,-7.58547 1.64425,0.87996 3.03048,0.77561 6.12039,-0.4607 9.36818,-3.74841 27.57182,-0.28312 38.6877,7.36493 4.01273,2.76084 5.02447,3.06807 6.68616,2.03033 4.46313,-2.78729 10.26008,5.35994 6.65117,9.34774 -1.39422,1.54059 -1.49005,2.41951 -0.55593,5.09915 2.35861,6.76594 2.19067,7.54117 -9.19719,42.4586 -7.29548,22.36927 -10.41136,29.00105 -14.25568,30.34117 -4.84484,1.68893 -19.66584,1.18599 -27.78786,-0.94298 z m 25.61313,-6.81949 c 1.38845,-2.06754 2.4589,-5.75927 2.92096,-10.07371 0.8462,-7.90176 0.92022,-7.80273 -9.60753,-12.85788 -7.91278,-3.79953 -19.49312,-4.89878 -25.43829,-2.41473 -3.08678,1.28974 -3.78835,2.27725 -5.40041,7.60156 -2.83618,9.36726 -1.07043,14.97521 5.33027,16.92851 3.93038,1.19945 24.82725,4.42665 27.50252,4.24734 1.57343,-0.10546 3.31356,-1.37784 4.69248,-3.43109 z m -9.28964,-38.51032 c 0.5394,-1.01741 0.70379,-2.59638 0.70379,-2.59638 1.41524,0.29516 2.39002,0.38257 3.12827,0.49034 1.14101,0.16658 2.52021,-0.39571 3.06493,-1.24949 1.21271,-1.90079 -3.82243,-10.82914 -6.60518,-11.71237 -2.64706,-0.84015 -10.86332,4.26659 -10.86332,6.75197 0,1.10878 1.85851,2.27842 2.69459,2.59926 0.45458,0.1745 1.26598,0.66101 2.24108,1.2639 -0.58959,1.24169 -0.69309,1.6233 -0.79538,2.36082 -0.32325,2.32767 0.37006,2.89195 3.32094,3.57097 2.20761,0.23784 2.50606,-0.19983 3.11028,-1.47902 z m 23.4929,-16.78596 c 11.02664,-10.10515 -0.57486,-28.33723 -13.70957,-21.54501 -3.24367,1.6774 -6.16684,7.06274 -6.16684,11.36122 0,6.65858 6.54586,13.49263 12.93598,13.50552 2.33286,0.006 4.37952,-0.97484 6.94043,-3.32173 z m -30.43621,-3.64967 c 5.69123,-3.47049 8.00525,-11.50538 4.99179,-17.3328 -2.16562,-4.1878 -5.39421,-6.09169 -10.33023,-6.09169 -11.59931,0 -17.68103,16.24444 -8.72051,23.29281 3.32212,2.61316 9.88873,2.67469 14.05895,0.13174 z" + id="path2370" + sodipodi:nodetypes="sscssscsssssssssssssssccssssscsccsscssssssss" /> + <path + id="path2360" + style="fill:#359136;stroke-width:1.01342" + d="m -47.976824,-62.377117 -2.300975,3.760736 c -1.262549,2.067224 -3.177544,6.496721 -4.261754,9.841005 -1.084213,3.344284 -2.654871,7.902195 -3.488579,10.13172 -0.833712,2.229523 -1.537357,7.472977 -1.564913,11.653332 -0.05867,8.894629 -1.199898,13.632786 -4.422575,18.3645139 -2.070761,3.0404069 -2.783527,3.3929217 -6.321501,3.1421954 -4.498771,-0.3188122 -7.638027,1.9759904 -12.265687,8.9688595 -1.449357,2.1901196 -3.128089,3.9834095 -3.72981,3.9834095 -0.601721,0 -2.16703,-1.2568269 -3.482391,-2.7896236 -1.315361,-1.5328538 -2.118968,-2.2322589 -1.781401,-1.5587245 1.097492,2.1897712 -1.507246,18.6086188 -3.686513,23.2448098 -2.521232,5.363696 -7.095667,9.129046 -12.457437,10.243057 -4.42862,0.920124 -3.78481,0.01756 -11.05335,15.457368 -5.9491,12.637045 -7.95853,14.598446 -17.55422,17.15217 -3.35344,0.892462 -6.36015,3.270257 -7.68848,5.783369 v 7.088492 c 1.03135,-0.243997 2.39957,-0.63556 4.67618,-1.354606 3.9954,-1.26191 9.29012,-2.79736 11.76467,-3.414352 5.93933,-1.480897 10.75791,-6.358877 12.29043,-12.445066 0.92546,-3.675352 2.25983,-5.623061 6.01223,-8.764744 3.72001,-3.114573 5.08815,-5.09418 5.98749,-8.665774 0.88852,-3.528634 1.83589,-4.914349 3.977224,-5.832852 4.559323,-1.955688 15.067434,-11.849394 16.781047,-15.797565 0.871585,-2.008131 2.558133,-6.312038 3.742178,-9.562659 2.127373,-5.840409 7.212024,-12.8347476 9.327616,-12.8347476 0.600125,0 3.934591,-1.8550895 7.410132,-4.11949272 8.478773,-5.52412988 9.525549,-7.95108558 9.525549,-21.99535468 0,-9.810853 0.254435,-11.432767 2.164899,-13.861528 1.901915,-2.417899 2.180001,-4.145098 2.282419,-14.282138 z M -73.57828,53.611896 c -0.376703,0.05474 -0.752942,0.203669 -1.181415,0.43298 -2.855303,1.528103 -3.799458,3.622053 -6.742108,14.962532 -1.677564,6.465026 -3.405321,10.785325 -4.97308,12.43888 -1.321154,1.393452 -2.726098,3.900924 -3.123638,5.573066 -1.795009,7.550278 -5.951929,12.905274 -12.599699,16.236726 -5.77361,2.89339 -11.02815,7.4995 -15.45737,13.55226 -2.28723,3.12562 -4.70081,6.01348 -5.36894,6.42047 -1.43366,0.87332 -4.4816,8.43236 -7.66993,19.02635 -2.59705,8.62936 -8.38879,15.85838 -11.95642,14.92542 -0.64738,-0.16929 -1.07597,-0.1606 -1.38553,0.3835 v 22.21803 c 1.61108,-2.55458 3.57363,-5.74565 7.42251,-12.19147 5.15871,-8.63939 11.29348,-18.67546 13.63266,-22.29843 2.33919,-3.62298 6.03751,-9.76 8.21424,-13.63886 3.25904,-5.8075 5.77046,-8.53752 14.24503,-15.45118 l 10.292536,-8.39362 5.276165,-15.96457 c 8.061735,-24.392548 10.288401,-27.113563 21.302588,-26.028251 4.916645,0.484478 8.312718,0.210063 12.482182,-1.002037 6.107242,-1.775429 6.968129,-3.207054 3.711252,-6.154493 -2.025968,-1.833294 -5.906107,-1.825776 -14.814085,0.03092 -2.603512,0.542658 -3.777159,0.1045 -6.791591,-2.542206 -2.282031,-2.003645 -3.385245,-2.700231 -4.515357,-2.536022 z m 21.376815,60.666604 c -0.162477,0.0465 -0.360651,0.16275 -0.5938,0.34639 -0.696726,0.54876 -2.48424,1.28914 -3.977225,1.64532 -2.172499,0.51828 -3.150699,1.81342 -4.892668,6.47613 -1.197118,3.20433 -2.769919,6.51222 -3.494764,7.34829 -0.724845,0.83606 -2.218581,3.59321 -3.321571,6.12975 -3.335074,7.66963 -7.429238,11.38002 -17.931537,16.25528 -9.527469,4.42276 -9.653804,4.52447 -12.259502,10.29873 -1.634118,3.62123 -2.860458,8.56542 -3.234978,13.03267 -0.80122,9.55692 -2.7613,11.8038 -14.56048,16.69447 -10.44173,4.32806 -17.2112,9.653 -17.67793,13.90481 -0.50794,4.62717 -2.36764,6.0497 -7.19983,5.50505 -1.12846,-0.12722 -1.9832,-0.17269 -2.69066,-0.0804 v 18.87791 c 2.97245,-4.14067 7.87345,-8.14292 18.02432,-14.78933 11.97418,-7.84026 18.76727,-14.28862 21.29021,-20.21395 1.00802,-2.36741 3.11806,-6.97654 4.68855,-10.24308 1.570494,-3.26656 3.148296,-7.5076 3.507138,-9.42039 0.938053,-5.00028 5.486535,-9.04582 12.327543,-10.96675 13.061728,-3.66769 14.7097,-5.27484 16.465592,-16.01405 1.457924,-8.91678 6.542344,-19.95882 12.086311,-26.25711 2.266557,-2.57494 4.119493,-5.69086 4.119493,-6.92767 0,-1.24166 -0.186786,-1.7415 -0.674212,-1.60203 z" /> + <path + style="fill:#30274f;stroke-width:1.01342" + d="m -18.083426,76.040966 c -4.247807,-1.108215 -4.537747,-1.397325 -4.230145,-4.217915 0.181623,-1.665472 0.409655,-3.099138 0.506709,-3.185905 0.0971,-0.08674 2.940611,0.521971 6.319006,1.352763 5.484847,1.348791 6.5425432,1.343379 9.8785677,-0.05048 4.79101285,-2.001817 6.52158792,-5.506332 4.78700985,-9.693974 C -1.7761088,57.942715 -3.8964069,56.299262 -9.4602217,53.550118 -18.996374,48.838225 -21.301368,46.348019 -21.832387,40.183785 c -0.790996,-9.181852 5.336229,-14.474323 16.7572435,-14.474323 3.3631019,0 7.2536093,0.432993 8.6455598,0.96221 2.31769,0.88119 2.4355919,1.214494 1.40008,3.957881 -0.9881346,2.617874 -1.4350937,2.879935 -3.5442315,2.078035 -4.0789528,-1.550804 -10.9769627,-1.080232 -13.4435838,0.917116 -2.521096,2.041451 -3.004931,6.389191 -0.982238,8.826385 0.705948,0.850614 4.92981,3.443568 9.3863501,5.762119 7.0658353,3.676049 8.3503634,4.775172 10.0371176,8.588356 2.5311668,5.722129 1.6259205,10.86357 -2.6704994,15.167369 -5.0767876,5.085512 -12.5853415,6.485672 -21.8368373,4.072033 z m 43.256589,-0.05371 c -7.668322,-2.857124 -10.82717,-7.916592 -10.82717,-17.341664 0,-7.136863 1.753045,-11.327587 6.471535,-15.470474 3.242818,-2.847243 4.467295,-3.277784 9.322058,-3.277784 3.090952,0 6.960795,0.709453 8.658434,1.587337 3.768327,1.948676 6.911133,8.456409 6.94172,14.374018 l 0.02217,4.307032 H 33.491012 21.220115 l 0.652753,2.786903 c 0.359004,1.532797 2.060849,4.084859 3.781878,5.671256 2.930008,2.700795 3.588731,2.86202 10.35137,2.533549 6.719759,-0.3264 7.243595,-0.201132 7.529177,1.800399 0.370557,2.597103 -1.235935,3.32057 -8.837207,3.979749 -3.559252,0.308649 -7.091982,-0.04377 -9.525008,-0.950321 z m 12.769369,-24.4356 C 37.667991,47.736952 34.36659,44.96443 30.098787,44.96443 c -3.316323,0 -5.350316,1.572978 -7.562085,5.848107 -1.900972,3.674396 -0.518649,4.391228 7.855856,4.073825 l 7.76879,-0.294449 z M 57.242759,75.96355 c -3.776264,-1.697905 -5.571283,-4.62028 -5.703524,-9.285629 -0.208543,-7.35719 5.12183,-11.782732 15.798549,-13.116696 5.360256,-0.669718 5.791995,-0.910143 5.189993,-2.890161 -1.6479,-5.4201 -7.296275,-7.27924 -14.17262,-4.664858 -3.10631,1.181016 -2.989099,1.23252 -3.892442,-1.710298 -0.562761,-1.833288 -0.110178,-2.374017 2.787483,-3.33033 5.272162,-1.739971 14.479257,-1.551282 17.390761,0.35641 4.483731,2.937852 5.388078,6.268303 5.748225,21.169013 l 0.335778,13.892307 -3.293613,-0.001 c -2.251403,-0.0014 -3.293613,-0.482885 -3.293613,-1.521994 0,-1.913852 -1.001167,-1.92266 -3.699709,-0.03253 -3.216869,2.253177 -9.506077,2.794514 -13.195268,1.135771 z m 13.601364,-7.32218 c 1.581137,-1.739208 2.280193,-3.774002 2.280193,-6.637124 v -4.128955 l -4.623491,0.521132 c -8.503979,0.95852 -12.329577,7.554442 -6.871591,11.847692 3.176715,2.498798 5.919898,2.02168 9.214889,-1.602745 z m 85.169137,7.750806 c -7.46757,-2.251048 -11.94959,-8.927284 -11.94959,-17.799647 0,-11.409471 6.80287,-18.769752 17.42431,-18.852023 8.41842,-0.06518 13.73789,5.513134 14.23491,14.927686 l 0.26349,4.990818 -11.14761,0.267828 c -6.13119,0.147294 -11.70114,0.468684 -12.37767,0.714166 -1.88359,0.683491 0.39212,6.206677 3.61151,8.765165 2.3428,1.861855 3.61058,2.066087 9.93121,1.599876 7.00108,-0.51641 7.26099,-0.453606 7.70357,1.861537 0.41161,2.153225 -0.0453,2.503919 -4.50854,3.461132 -5.70006,1.222437 -9.28361,1.239687 -13.18559,0.06343 z m 12.37248,-23.690432 c 0,-4.037403 -3.69287,-7.737314 -7.7226,-7.737314 -4.13657,0 -8.49211,3.974215 -8.49211,7.748623 0,2.306352 0.26922,2.38557 8.10735,2.38557 7.84897,0 8.10736,-0.07642 8.10736,-2.396879 z m 25.05236,23.466232 c -8.07572,-2.143056 -12.67959,-10.441554 -11.41373,-20.573233 1.29815,-10.390063 7.07629,-15.687206 17.11864,-15.693619 5.35707,-0.003 6.25058,0.318273 9.26966,3.33737 3.35578,3.355758 5.56075,8.952214 5.57127,14.140323 l 0.006,2.786903 h -12.16103 c -11.61442,0 -12.16103,0.09406 -12.16103,2.093461 0,6.165197 5.65846,9.52822 14.79738,8.794634 5.76331,-0.462626 6.36072,-0.322894 6.86506,1.605624 0.68811,2.63127 0.62623,2.688399 -4.10068,3.785435 -4.81839,1.118276 -8.83787,1.037567 -13.79121,-0.276886 z m 12.60687,-24.109611 c -0.5292,-3.89762 -4.06612,-7.093935 -7.84992,-7.093935 -3.67013,0 -8.52751,4.685686 -8.52751,8.226095 0,1.840788 0.62718,1.967991 8.36071,1.695727 8.26947,-0.291137 8.35698,-0.321986 8.01672,-2.827887 z m 22.35362,23.650085 c -4.29031,-2.11461 -6.97387,-5.951333 -8.2993,-11.865547 -1.90649,-8.507273 0.37503,-16.480198 6.09271,-21.291305 4.14115,-3.48456 12.46341,-3.76319 16.54337,-0.553871 l 2.67024,2.10039 V 33.890372 23.682623 h 4.05367 4.05368 V 50.031526 76.38043 h -3.54697 c -3.16946,0 -3.54696,-0.288445 -3.54696,-2.710197 v -2.71021 l -2.2802,2.135984 c -4.66496,4.369937 -10.28275,5.302334 -15.74024,2.612443 z m 14.0447,-7.357424 c 2.72201,-2.722003 2.96232,-3.550382 2.96232,-10.212145 0,-6.725396 -0.22295,-7.47272 -3.08086,-10.330705 -2.57605,-2.576033 -3.61392,-2.978582 -6.33387,-2.45668 -9.14754,1.75521 -11.86467,17.777177 -4.0564,23.919148 3.62893,2.854507 7.03256,2.556663 10.50881,-0.919618 z M 92.550115,58.834533 c -2.880857,-9.650244 -5.237928,-17.858941 -5.237928,-18.241548 0,-0.382607 1.745726,-0.695652 3.879401,-0.695652 h 3.879398 l 3.379937,14.441226 c 1.858957,7.942674 3.528677,13.985187 3.710477,13.427806 0.18178,-0.557381 2.09943,-6.941923 4.26143,-14.187871 l 3.93087,-13.174452 h 3.32821 3.3282 l 3.49617,12.667742 c 1.9229,6.967258 3.72739,13.3518 4.00999,14.187871 0.28262,0.836071 2.12128,-4.978422 4.08597,-12.921096 l 3.57214,-14.441226 h 4.02623 c 3.11186,0 3.8974,0.335727 3.45896,1.478284 -0.312,0.813047 -2.9528,9.021744 -5.86843,18.241549 l -5.30118,16.763264 h -3.67487 -3.67487 l -3.34183,-11.907678 c -1.83799,-6.549222 -3.57389,-12.591735 -3.85754,-13.427806 -0.57098,-1.682995 -0.27096,-2.483099 -5.41523,14.441226 l -3.31139,10.894258 h -3.71309 -3.713085 z" + id="path2354" + sodipodi:nodetypes="sssssssssssssssssscssssccccsssssccsssscssssssssssccsssssscssssssscssssssssssssscscsccsssssccssssssssscccccccsscssssssssssscssscccssscssscccssscccs" /> + <path + style="fill:url(#radialGradient19627);fill-opacity:1;stroke-width:1.01342" + d="m -75.969113,-64.844727 c -9.759799,-2.838682 -12.419771,-15.328145 -4.816086,-22.612951 8.10707,-7.767059 20.740989,-1.606264 20.740989,10.114118 0,8.688342 -7.848109,14.848009 -15.924903,12.498833 z" + id="path1933-7-6-8-6" + sodipodi:nodetypes="ssss" /> + <path + style="fill:url(#radialGradient21593);fill-opacity:1;stroke-width:1.01342" + d="m -86.660719,212.12738 c -2.88245,-3.47311 1.093004,-9.34439 4.767441,-7.04101 3.616229,2.26689 2.31541,8.50537 -1.773484,8.50537 -0.978266,0 -2.325544,-0.65894 -2.993957,-1.46436 z" + id="path1933-7" + sodipodi:nodetypes="cssc" /> + <path + style="fill:url(#radialGradient21591);fill-opacity:1;stroke-width:1.01342" + d="m -75.052697,192.15327 c -5.173443,-2.82966 -6.112407,-8.51751 -2.072918,-12.557 2.823418,-2.82341 6.203679,-3.18236 9.332262,-0.99103 4.829545,3.38277 3.652174,11.69223 -1.932464,13.63904 0,0 -3.40748,0.93944 -5.32688,-0.0909 z" + id="path1933-7-6" + sodipodi:nodetypes="cssccc" /> + <path + style="fill:url(#radialGradient21589);fill-opacity:1;stroke-width:1.01342" + d="m -123.24459,-1.9528648 c -4.3229,-4.3228923 -0.83806,-13.1744512 5.18671,-13.1744512 4.51503,0 8.39807,4.282102 7.58224,8.361387 -0.3476,1.7379635 -1.38132,3.9879286 -2.29717,4.9999197 -2.25733,2.49432909 -7.89114,2.39379789 -10.47178,-0.1868492 z" + id="path1933-7-6-8" + sodipodi:nodetypes="ssssss" /> + <path + id="path18907" + style="opacity:1;stroke-width:3.16694" + d="m -45.140154,214.41827 a 3.4737418,3.4737418 0 0 1 -3.473741,3.47375 3.4737418,3.4737418 0 0 1 -3.473742,-3.47375 3.4737418,3.4737418 0 0 1 3.473742,-3.47374 3.4737418,3.4737418 0 0 1 3.473741,3.47374 z" /> + <path + style="color:#000000;fill:#b3b3b3;stroke-width:3.16694;-inkscape-stroke:none" + d="m -123.15443,-148.31736 c -13.95653,0 -25.26744,11.31091 -25.26744,25.26745 v 386.05687 c 0,13.95652 11.31091,25.26742 25.26744,25.26742 h 386.47127 c 13.95653,0 25.26129,-11.31166 25.26129,-25.26742 v -386.05687 c 0,-13.95577 -11.30476,-25.26745 -25.26129,-25.26745 z m 0,4.75041 h 386.47127 c 11.40667,0 20.51088,9.1096 20.51088,20.51704 v 386.05687 c 0,11.40743 -9.10421,20.51702 -20.51088,20.51702 h -386.47127 c -11.40667,0 -20.51704,-9.11035 -20.51704,-20.51702 v -386.05687 c 0,-11.40667 9.11037,-20.51704 20.51704,-20.51704 z" + id="path28825" /> + <path + id="path1004" + style="fill:#59686f" + d="m 216.21641,209.1283 a 4.4553308,3.9311745 0 0 1 -4.45533,3.93117 4.4553308,3.9311745 0 0 1 -4.45533,-3.93117 4.4553308,3.9311745 0 0 1 4.45533,-3.93118 4.4553308,3.9311745 0 0 1 4.45533,3.93118 z" /> + </g> +</svg> diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index b0829163c..7e948f0dc 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -54,6 +54,9 @@ service SeaweedFiler { rpc GetFilerConfiguration (GetFilerConfigurationRequest) returns (GetFilerConfigurationResponse) { } + rpc TraverseBfsMetadata (TraverseBfsMetadataRequest) returns (stream TraverseBfsMetadataResponse) { + } + rpc SubscribeMetadata (SubscribeMetadataRequest) returns (stream SubscribeMetadataResponse) { } @@ -218,6 +221,7 @@ message DeleteEntryRequest { bool ignore_recursive_error = 6; bool is_from_other_cluster = 7; repeated int32 signatures = 8; + int64 if_not_modified_after = 9; } message DeleteEntryResponse { @@ -341,6 +345,8 @@ message GetFilerConfigurationResponse { string version = 11; string cluster_id = 12; string filer_group = 13; + int32 major_version = 14; + int32 minor_version = 15; } message SubscribeMetadataRequest { @@ -360,6 +366,15 @@ message SubscribeMetadataResponse { int64 ts_ns = 3; } +message TraverseBfsMetadataRequest { + string directory = 1; + repeated string excluded_prefixes = 2; +} +message TraverseBfsMetadataResponse { + string directory = 1; + Entry entry = 2; +} + message LogEntry { int64 ts_ns = 1; int32 partition_key_hash = 2; diff --git a/unmaintained/change_superblock/change_superblock.go b/unmaintained/change_superblock/change_superblock.go index 27876272c..52368f8cd 100644 --- a/unmaintained/change_superblock/change_superblock.go +++ b/unmaintained/change_superblock/change_superblock.go @@ -11,6 +11,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/storage/backend" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/super_block" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var ( @@ -40,6 +41,8 @@ that has those volumes. */ func main() { flag.Parse() + util_http.NewGlobalHttpClient() + fileName := strconv.Itoa(*fixVolumeId) if *fixVolumeCollection != "" { fileName = *fixVolumeCollection + "_" + fileName diff --git a/unmaintained/diff_volume_servers/diff_volume_servers.go b/unmaintained/diff_volume_servers/diff_volume_servers.go index 698d7dd3b..e289fefe8 100644 --- a/unmaintained/diff_volume_servers/diff_volume_servers.go +++ b/unmaintained/diff_volume_servers/diff_volume_servers.go @@ -20,6 +20,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/storage/types" "github.com/seaweedfs/seaweedfs/weed/util" "google.golang.org/grpc" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var ( @@ -40,8 +41,9 @@ var ( */ func main() { flag.Parse() - - util.LoadConfiguration("security", false) + util_http.InitGlobalHttpClient() + + util.LoadSecurityConfiguration() grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client") vid := uint32(*volumeId) diff --git a/unmaintained/fix_dat/fix_dat.go b/unmaintained/fix_dat/fix_dat.go index 760fc79ca..164b5b238 100644 --- a/unmaintained/fix_dat/fix_dat.go +++ b/unmaintained/fix_dat/fix_dat.go @@ -14,6 +14,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/storage/super_block" "github.com/seaweedfs/seaweedfs/weed/storage/types" "github.com/seaweedfs/seaweedfs/weed/util" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var ( @@ -36,6 +37,8 @@ The .idx has all correct offsets. */ func main() { flag.Parse() + util_http.InitGlobalHttpClient() + fileName := strconv.Itoa(*fixVolumeId) if *fixVolumeCollection != "" { fileName = *fixVolumeCollection + "_" + fileName diff --git a/unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go b/unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go index eb08e5353..2b63d5d59 100644 --- a/unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go +++ b/unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go @@ -10,7 +10,9 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" "strconv" + "strings" "time" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var ( @@ -22,8 +24,8 @@ var ( ) func main() { - flag.Parse() + util_http.InitGlobalHttpClient() if *isWrite { startGenerateMetadata() @@ -79,12 +81,17 @@ func startGenerateMetadata() { func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) { + prefix := *dir + if !strings.HasSuffix(prefix, "/") { + prefix = prefix + "/" + } + metadataFollowOption := &pb.MetadataFollowOption{ ClientName: "tail", ClientId: 0, ClientEpoch: 0, SelfSignature: 0, - PathPrefix: *dir, + PathPrefix: prefix, AdditionalPathPrefixes: nil, DirectoriesToWatch: nil, StartTsNs: 0, diff --git a/unmaintained/remove_duplicate_fids/remove_duplicate_fids.go b/unmaintained/remove_duplicate_fids/remove_duplicate_fids.go index 354707c81..cfac97432 100644 --- a/unmaintained/remove_duplicate_fids/remove_duplicate_fids.go +++ b/unmaintained/remove_duplicate_fids/remove_duplicate_fids.go @@ -11,6 +11,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/storage/backend" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/super_block" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var ( @@ -71,6 +72,7 @@ func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *needle.Needle, offset in func main() { flag.Parse() + util_http.InitGlobalHttpClient() vid := needle.VolumeId(*volumeId) diff --git a/unmaintained/repeated_vacuum/repeated_vacuum.go b/unmaintained/repeated_vacuum/repeated_vacuum.go index 893f4a68a..1f89bd902 100644 --- a/unmaintained/repeated_vacuum/repeated_vacuum.go +++ b/unmaintained/repeated_vacuum/repeated_vacuum.go @@ -14,6 +14,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/util" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var ( @@ -25,8 +26,9 @@ var ( func main() { flag.Parse() + util_http.InitGlobalHttpClient() - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") genFile(grpcDialOption, 0) @@ -34,7 +36,7 @@ func main() { go func() { for { println("vacuum threshold", *garbageThreshold) - _, _, err := util.Get(fmt.Sprintf("http://%s/vol/vacuum?garbageThreshold=%f", pb.ServerAddress(*master).ToHttpAddress(), *garbageThreshold)) + _, _, err := util_http.Get(fmt.Sprintf("http://%s/vol/vacuum?garbageThreshold=%f", pb.ServerAddress(*master).ToHttpAddress(), *garbageThreshold)) if err != nil { log.Fatalf("vacuum: %v", err) } @@ -47,7 +49,7 @@ func main() { assignResult, targetUrl := genFile(grpcDialOption, i) - util.Delete(targetUrl, string(assignResult.Auth)) + util_http.Delete(targetUrl, string(assignResult.Auth)) } @@ -76,7 +78,13 @@ func genFile(grpcDialOption grpc.DialOption, i int) (*operation.AssignResult, st PairMap: nil, Jwt: assignResult.Auth, } - _, err = operation.UploadData(data, uploadOption) + + uploader, err := operation.NewUploader() + if err != nil { + log.Fatalf("upload: %v", err) + } + + _, err = uploader.UploadData(data, uploadOption) if err != nil { log.Fatalf("upload: %v", err) } diff --git a/unmaintained/s3/presigned_put/presigned_put.go b/unmaintained/s3/presigned_put/presigned_put.go index ba135ff25..1e591dff2 100644 --- a/unmaintained/s3/presigned_put/presigned_put.go +++ b/unmaintained/s3/presigned_put/presigned_put.go @@ -7,10 +7,10 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" - "github.com/seaweedfs/seaweedfs/weed/util" "net/http" "strings" "time" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) // Downloads an item from an S3 Bucket in the region configured in the shared config @@ -21,6 +21,8 @@ import ( // For this exampl to work, the domainName is needd // weed s3 -domainName=localhost func main() { + util_http.InitGlobalHttpClient() + h := md5.New() content := strings.NewReader(stringContent) content.WriteTo(h) @@ -64,7 +66,7 @@ func main() { fmt.Printf("error put request: %v\n", err) return } - defer util.CloseResponse(resp) + defer util_http.CloseResponse(resp) fmt.Printf("response: %+v\n", resp) } diff --git a/unmaintained/see_dat/see_dat.go b/unmaintained/see_dat/see_dat.go index 1b2f0bb6a..a60e45760 100644 --- a/unmaintained/see_dat/see_dat.go +++ b/unmaintained/see_dat/see_dat.go @@ -10,6 +10,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/storage" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/super_block" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var ( @@ -40,6 +41,7 @@ func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *needle.Needle, offset in func main() { flag.Parse() + util_http.InitGlobalHttpClient() vid := needle.VolumeId(*volumeId) diff --git a/unmaintained/see_idx/see_idx.go b/unmaintained/see_idx/see_idx.go index 856d96d54..87f00ebb0 100644 --- a/unmaintained/see_idx/see_idx.go +++ b/unmaintained/see_idx/see_idx.go @@ -12,6 +12,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/storage/idx" "github.com/seaweedfs/seaweedfs/weed/storage/types" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var ( @@ -27,6 +28,8 @@ This is to see content in .idx files. */ func main() { flag.Parse() + util_http.InitGlobalHttpClient() + fileName := strconv.Itoa(*fixVolumeId) if *fixVolumeCollection != "" { fileName = *fixVolumeCollection + "_" + fileName diff --git a/unmaintained/see_log_entry/see_log_entry.go b/unmaintained/see_log_entry/see_log_entry.go index d5deff283..42a63476b 100644 --- a/unmaintained/see_log_entry/see_log_entry.go +++ b/unmaintained/see_log_entry/see_log_entry.go @@ -12,6 +12,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/util" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var ( @@ -20,6 +21,7 @@ var ( func main() { flag.Parse() + util_http.InitGlobalHttpClient() dst, err := os.OpenFile(*logdataFile, os.O_RDONLY, 0644) if err != nil { diff --git a/unmaintained/see_meta/see_meta.go b/unmaintained/see_meta/see_meta.go index 6fc88358c..da78f0918 100644 --- a/unmaintained/see_meta/see_meta.go +++ b/unmaintained/see_meta/see_meta.go @@ -11,6 +11,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/util" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var ( @@ -19,6 +20,7 @@ var ( func main() { flag.Parse() + util_http.InitGlobalHttpClient() dst, err := os.OpenFile(*metaFile, os.O_RDONLY, 0644) if err != nil { diff --git a/unmaintained/stream_read_volume/stream_read_volume.go b/unmaintained/stream_read_volume/stream_read_volume.go index cdbfa2c44..cfdb36815 100644 --- a/unmaintained/stream_read_volume/stream_read_volume.go +++ b/unmaintained/stream_read_volume/stream_read_volume.go @@ -13,6 +13,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/util" "google.golang.org/grpc" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var ( @@ -23,8 +24,9 @@ var ( func main() { flag.Parse() + util_http.InitGlobalHttpClient() - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client") vid := uint32(*volumeId) diff --git a/unmaintained/stress_filer_upload/bench_filer_upload/bench_filer_upload.go b/unmaintained/stress_filer_upload/bench_filer_upload/bench_filer_upload.go index 1767b04b2..6dc703dbc 100644 --- a/unmaintained/stress_filer_upload/bench_filer_upload/bench_filer_upload.go +++ b/unmaintained/stress_filer_upload/bench_filer_upload/bench_filer_upload.go @@ -13,6 +13,7 @@ import ( "strings" "sync" "time" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var ( @@ -30,8 +31,8 @@ type stat struct { } func main() { - flag.Parse() + util_http.InitGlobalHttpClient() data := make([]byte, *size) println("data len", len(data)) @@ -43,16 +44,12 @@ func main() { go func(x int) { defer wg.Done() - client := &http.Client{Transport: &http.Transport{ - MaxIdleConns: 1024, - MaxIdleConnsPerHost: 1024, - }} r := rand.New(rand.NewSource(time.Now().UnixNano() + int64(x))) for t := 0; t < *times; t++ { for f := 0; f < *fileCount; f++ { fn := r.Intn(*fileCount) - if size, err := uploadFileToFiler(client, data, fmt.Sprintf("file%04d", fn), *destination); err == nil { + if size, err := uploadFileToFiler(data, fmt.Sprintf("file%04d", fn), *destination); err == nil { statsChan <- stat{ size: size, } @@ -93,7 +90,7 @@ func main() { } -func uploadFileToFiler(client *http.Client, data []byte, filename, destination string) (size int64, err error) { +func uploadFileToFiler(data []byte, filename, destination string) (size int64, err error) { if !strings.HasSuffix(destination, "/") { destination = destination + "/" @@ -115,11 +112,14 @@ func uploadFileToFiler(client *http.Client, data []byte, filename, destination s uri := destination + filename - request, err := http.NewRequest("POST", uri, body) + request, err := http.NewRequest(http.MethodPost, uri, body) + if err != nil { + return 0, fmt.Errorf("http POST %s: %v", uri, err) + } request.Header.Set("Content-Type", writer.FormDataContentType()) // request.Close = true // can not use this, which do not reuse http connection, impacting filer->volume also. - resp, err := client.Do(request) + resp, err := util_http.GetGlobalHttpClient().Do(request) if err != nil { return 0, fmt.Errorf("http POST %s: %v", uri, err) } else { diff --git a/unmaintained/stress_filer_upload/stress_filer_upload_actual/stress_filer_upload.go b/unmaintained/stress_filer_upload/stress_filer_upload_actual/stress_filer_upload.go index afcd51917..1cdcad0b3 100644 --- a/unmaintained/stress_filer_upload/stress_filer_upload_actual/stress_filer_upload.go +++ b/unmaintained/stress_filer_upload/stress_filer_upload_actual/stress_filer_upload.go @@ -14,6 +14,7 @@ import ( "strings" "sync" "time" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var ( @@ -30,8 +31,8 @@ type stat struct { } func main() { - flag.Parse() + util_http.InitGlobalHttpClient() var fileNames []string @@ -51,8 +52,6 @@ func main() { for x := 0; x < *concurrency; x++ { wg.Add(1) - client := &http.Client{} - go func() { defer wg.Done() rand.Shuffle(len(fileNames), func(i, j int) { @@ -60,7 +59,7 @@ func main() { }) for t := 0; t < *times; t++ { for _, filename := range fileNames { - if size, err := uploadFileToFiler(client, filename, *destination); err == nil { + if size, err := uploadFileToFiler(filename, *destination); err == nil { statsChan <- stat{ size: size, } @@ -99,7 +98,7 @@ func main() { } -func uploadFileToFiler(client *http.Client, filename, destination string) (size int64, err error) { +func uploadFileToFiler(filename, destination string) (size int64, err error) { file, err := os.Open(filename) if err != nil { panic(err) @@ -130,10 +129,14 @@ func uploadFileToFiler(client *http.Client, filename, destination string) (size uri := destination + file.Name() - request, err := http.NewRequest("POST", uri, body) + request, err := http.NewRequest(http.MethodPost, uri, body) + if err != nil { + return 0, fmt.Errorf("http POST %s: %v", uri, err) + } + request.Header.Set("Content-Type", writer.FormDataContentType()) - resp, err := client.Do(request) + resp, err := util_http.GetGlobalHttpClient().Do(request) if err != nil { return 0, fmt.Errorf("http POST %s: %v", uri, err) } else { diff --git a/unmaintained/volume_tailer/volume_tailer.go b/unmaintained/volume_tailer/volume_tailer.go index 50ef1c74f..a75a095d4 100644 --- a/unmaintained/volume_tailer/volume_tailer.go +++ b/unmaintained/volume_tailer/volume_tailer.go @@ -12,6 +12,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/storage/needle" util2 "github.com/seaweedfs/seaweedfs/weed/util" "golang.org/x/tools/godoc/util" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var ( @@ -24,8 +25,9 @@ var ( func main() { flag.Parse() + util_http.InitGlobalHttpClient() - util2.LoadConfiguration("security", false) + util2.LoadSecurityConfiguration() grpcDialOption := security.LoadClientTLS(util2.GetViper(), "grpc.client") vid := needle.VolumeId(*volumeId) diff --git a/weed/command/backup.go b/weed/command/backup.go index a8be4838e..f9b9fba64 100644 --- a/weed/command/backup.go +++ b/weed/command/backup.go @@ -66,7 +66,7 @@ var cmdBackup = &Command{ func runBackup(cmd *Command, args []string) bool { - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") if *s.volumeId == -1 { @@ -138,7 +138,9 @@ func runBackup(cmd *Command, args []string) bool { if datSize > stats.TailOffset { // remove the old data - v.Destroy(false) + if err := v.Destroy(false); err != nil { + fmt.Printf("Error destroying volume: %v\n", err) + } // recreate an empty volume v, err = storage.NewVolume(util.ResolvePath(*s.dir), util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0, 0) if err != nil { diff --git a/weed/command/benchmark.go b/weed/command/benchmark.go index 2a0db47c2..bc7ee1292 100644 --- a/weed/command/benchmark.go +++ b/weed/command/benchmark.go @@ -22,6 +22,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/wdclient" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) type BenchmarkOptions struct { @@ -111,7 +112,7 @@ var ( func runBenchmark(cmd *Command, args []string) bool { - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() b.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client") fmt.Printf("This is SeaweedFS version %s %s %s\n", util.Version(), runtime.GOOS, runtime.GOARCH) @@ -214,7 +215,7 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) { if isSecure { jwtAuthorization = operation.LookupJwt(b.masterClient.GetMaster(context.Background()), b.grpcDialOption, df.fp.Fid) } - if e := util.Delete(fmt.Sprintf("http://%s/%s", df.fp.Server, df.fp.Fid), string(jwtAuthorization)); e == nil { + if e := util_http.Delete(fmt.Sprintf("http://%s/%s", df.fp.Server, df.fp.Fid), string(jwtAuthorization)); e == nil { s.completed++ } else { s.failed++ @@ -295,7 +296,7 @@ func readFiles(fileIdLineChan chan string, s *stat) { } var bytes []byte for _, url := range urls { - bytes, _, err = util.Get(url) + bytes, _, err = util_http.Get(url) if err == nil { break } diff --git a/weed/command/download.go b/weed/command/download.go index 060be9f14..1b7098824 100644 --- a/weed/command/download.go +++ b/weed/command/download.go @@ -15,6 +15,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/util" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var ( @@ -47,7 +48,7 @@ var cmdDownload = &Command{ } func runDownload(cmd *Command, args []string) bool { - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") for _, fid := range args { @@ -63,11 +64,11 @@ func downloadToFile(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpti if lookupError != nil { return lookupError } - filename, _, rc, err := util.DownloadFile(fileUrl, jwt) + filename, _, rc, err := util_http.DownloadFile(fileUrl, jwt) if err != nil { return err } - defer util.CloseResponse(rc) + defer util_http.CloseResponse(rc) if filename == "" { filename = fileId } @@ -116,10 +117,10 @@ func fetchContent(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOption return "", nil, lookupError } var rc *http.Response - if filename, _, rc, e = util.DownloadFile(fileUrl, jwt); e != nil { + if filename, _, rc, e = util_http.DownloadFile(fileUrl, jwt); e != nil { return "", nil, e } - defer util.CloseResponse(rc) + defer util_http.CloseResponse(rc) content, e = io.ReadAll(rc.Body) return } diff --git a/weed/command/filer.go b/weed/command/filer.go index 877c4b5d5..b7f67ea3b 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -1,6 +1,9 @@ package command import ( + "context" + "crypto/tls" + "crypto/x509" "fmt" "net" "net/http" @@ -10,8 +13,6 @@ import ( "strings" "time" - "google.golang.org/grpc/reflection" - "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb" @@ -20,6 +21,10 @@ import ( weed_server "github.com/seaweedfs/seaweedfs/weed/server" stats_collect "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/util" + "github.com/spf13/viper" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/credentials/tls/certprovider/pemfile" + "google.golang.org/grpc/reflection" ) var ( @@ -52,6 +57,7 @@ type FilerOptions struct { disableHttp *bool cipher *bool metricsHttpPort *int + metricsHttpIp *string saveToFilerLimit *int defaultLevelDbDirectory *string concurrentUploadLimitMB *int @@ -63,7 +69,7 @@ type FilerOptions struct { diskType *string allowedOrigins *string exposeDirectoryData *bool - joinExistingFiler *bool + certProvider certprovider.Provider } func init() { @@ -85,6 +91,7 @@ func init() { f.disableHttp = cmdFiler.Flag.Bool("disableHttp", false, "disable http request, only gRpc operations are allowed") f.cipher = cmdFiler.Flag.Bool("encryptVolumeData", false, "encrypt data on volume servers") f.metricsHttpPort = cmdFiler.Flag.Int("metricsPort", 0, "Prometheus metrics listen port") + f.metricsHttpIp = cmdFiler.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.") f.saveToFilerLimit = cmdFiler.Flag.Int("saveToFilerLimit", 0, "files smaller than this limit will be saved in filer store") f.defaultLevelDbDirectory = cmdFiler.Flag.String("defaultStoreDir", ".", "if filer.toml is empty, use an embedded filer store in the directory") f.concurrentUploadLimitMB = cmdFiler.Flag.Int("concurrentUploadLimitMB", 128, "limit total concurrent upload size") @@ -96,7 +103,6 @@ func init() { f.diskType = cmdFiler.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag") f.allowedOrigins = cmdFiler.Flag.String("allowedOrigins", "*", "comma separated list of allowed origins") f.exposeDirectoryData = cmdFiler.Flag.Bool("exposeDirectoryData", true, "whether to return directory metadata and content in Filer UI") - f.joinExistingFiler = cmdFiler.Flag.Bool("joinExistingFiler", false, "enable if new filer wants to join existing cluster") // start s3 on filer filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway") @@ -124,6 +130,7 @@ func init() { filerWebDavOptions.tlsCertificate = cmdFiler.Flag.String("webdav.cert.file", "", "path to the TLS certificate file") filerWebDavOptions.cacheDir = cmdFiler.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks") filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB") + filerWebDavOptions.maxMB = cmdFiler.Flag.Int("webdav.maxMB", 4, "split files larger than the limit") filerWebDavOptions.filerRootPath = cmdFiler.Flag.String("webdav.filer.path", "/", "use this remote path from filer server") // start iam on filer @@ -172,9 +179,17 @@ func runFiler(cmd *Command, args []string) bool { go http.ListenAndServe(fmt.Sprintf(":%d", *f.debugPort), nil) } - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() - go stats_collect.StartMetricsServer(*f.bindIp, *f.metricsHttpPort) + switch { + case *f.metricsHttpIp != "": + // noting to do, use f.metricsHttpIp + case *f.bindIp != "": + *f.metricsHttpIp = *f.bindIp + case *f.ip != "": + *f.metricsHttpIp = *f.ip + } + go stats_collect.StartMetricsServer(*f.metricsHttpIp, *f.metricsHttpPort) filerAddress := pb.NewServerAddress(*f.ip, *f.port, *f.portGrpc).String() startDelay := time.Duration(2) @@ -222,6 +237,15 @@ func runFiler(cmd *Command, args []string) bool { return true } +// GetCertificateWithUpdate Auto refreshing TSL certificate +func (fo *FilerOptions) GetCertificateWithUpdate(*tls.ClientHelloInfo) (*tls.Certificate, error) { + certs, err := fo.certProvider.KeyMaterial(context.Background()) + if certs == nil { + return nil, err + } + return &certs.Certs[0], err +} + func (fo *FilerOptions) startFiler() { defaultMux := http.NewServeMux() @@ -264,7 +288,6 @@ func (fo *FilerOptions) startFiler() { DownloadMaxBytesPs: int64(*fo.downloadMaxMBps) * 1024 * 1024, DiskType: *fo.diskType, AllowedOrigins: strings.Split(*fo.allowedOrigins, ","), - JoinExistingFiler: *fo.joinExistingFiler, }) if nfs_err != nil { glog.Fatalf("Filer startup error: %v", nfs_err) @@ -332,15 +355,62 @@ func (fo *FilerOptions) startFiler() { httpS.Serve(filerSocketListener) }() } - if filerLocalListener != nil { - go func() { - if err := httpS.Serve(filerLocalListener); err != nil { - glog.Errorf("Filer Fail to serve: %v", e) + + if viper.GetString("https.filer.key") != "" { + certFile := viper.GetString("https.filer.cert") + keyFile := viper.GetString("https.filer.key") + caCertFile := viper.GetString("https.filer.ca") + disbaleTlsVerifyClientCert := viper.GetBool("https.filer.disable_tls_verify_client_cert") + + pemfileOptions := pemfile.Options{ + CertFile: certFile, + KeyFile: keyFile, + RefreshDuration: security.CredRefreshingInterval, + } + if fo.certProvider, err = pemfile.NewProvider(pemfileOptions); err != nil { + glog.Fatalf("pemfile.NewProvider(%v) failed: %v", pemfileOptions, err) + } + + caCertPool := x509.NewCertPool() + if caCertFile != "" { + caCertFile, err := os.ReadFile(caCertFile) + if err != nil { + glog.Fatalf("error reading CA certificate: %v", err) } - }() - } - if err := httpS.Serve(filerListener); err != nil { - glog.Fatalf("Filer Fail to serve: %v", e) - } + caCertPool.AppendCertsFromPEM(caCertFile) + } + + clientAuth := tls.NoClientCert + if !disbaleTlsVerifyClientCert { + clientAuth = tls.RequireAndVerifyClientCert + } + + httpS.TLSConfig = &tls.Config{ + GetCertificate: fo.GetCertificateWithUpdate, + ClientAuth: clientAuth, + ClientCAs: caCertPool, + } + if filerLocalListener != nil { + go func() { + if err := httpS.ServeTLS(filerLocalListener, "", ""); err != nil { + glog.Errorf("Filer Fail to serve: %v", e) + } + }() + } + if err := httpS.ServeTLS(filerListener, "", ""); err != nil { + glog.Fatalf("Filer Fail to serve: %v", e) + } + } else { + if filerLocalListener != nil { + go func() { + if err := httpS.Serve(filerLocalListener); err != nil { + glog.Errorf("Filer Fail to serve: %v", e) + } + }() + } + if err := httpS.Serve(filerListener); err != nil { + glog.Fatalf("Filer Fail to serve: %v", e) + } + } } diff --git a/weed/command/filer_backup.go b/weed/command/filer_backup.go index 4aeab60f2..1344dfd2c 100644 --- a/weed/command/filer_backup.go +++ b/weed/command/filer_backup.go @@ -9,6 +9,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/util" "google.golang.org/grpc" "regexp" + "strings" "time" ) @@ -58,7 +59,7 @@ var cmdFilerBackup = &Command{ func runFilerBackup(cmd *Command, args []string) bool { - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() util.LoadConfiguration("replication", true) grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") @@ -148,17 +149,22 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti }() } + prefix := sourcePath + if !strings.HasSuffix(prefix, "/") { + prefix = prefix + "/" + } + metadataFollowOption := &pb.MetadataFollowOption{ ClientName: "backup_" + dataSink.GetName(), ClientId: clientId, ClientEpoch: clientEpoch, SelfSignature: 0, - PathPrefix: sourcePath, + PathPrefix: prefix, AdditionalPathPrefixes: nil, DirectoriesToWatch: nil, StartTsNs: startFrom.UnixNano(), StopTsNs: 0, - EventErrorType: pb.TrivialOnError, + EventErrorType: pb.RetryForeverOnError, } return pb.FollowMetadata(sourceFiler, grpcDialOption, metadataFollowOption, processEventFnWithOffset) diff --git a/weed/command/filer_cat.go b/weed/command/filer_cat.go index 2ef3bfc33..ba3625b0d 100644 --- a/weed/command/filer_cat.go +++ b/weed/command/filer_cat.go @@ -59,7 +59,7 @@ var cmdFilerCat = &Command{ func runFilerCat(cmd *Command, args []string) bool { - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() if len(args) == 0 { return false diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.go index df5e002c5..0342aa585 100644 --- a/weed/command/filer_copy.go +++ b/weed/command/filer_copy.go @@ -83,7 +83,7 @@ var cmdFilerCopy = &Command{ func runCopy(cmd *Command, args []string) bool { - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() if len(args) <= 1 { return false @@ -344,7 +344,12 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err return err } - finalFileId, uploadResult, flushErr, _ := operation.UploadWithRetry( + uploader, uploaderErr := operation.NewUploader() + if uploaderErr != nil { + return uploaderErr + } + + finalFileId, uploadResult, flushErr, _ := uploader.UploadWithRetry( worker, &filer_pb.AssignVolumeRequest{ Count: 1, @@ -423,7 +428,13 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File, <-concurrentChunks }() - fileId, uploadResult, err, _ := operation.UploadWithRetry( + uploader, err := operation.NewUploader() + if err != nil { + uploadError = fmt.Errorf("upload data %v: %v\n", fileName, err) + return + } + + fileId, uploadResult, err, _ := uploader.UploadWithRetry( worker, &filer_pb.AssignVolumeRequest{ Count: 1, @@ -472,7 +483,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File, for _, chunk := range chunks { fileIds = append(fileIds, chunk.FileId) } - operation.DeleteFiles(func(_ context.Context) pb.ServerAddress { + operation.DeleteFileIds(func(_ context.Context) pb.ServerAddress { return pb.ServerAddress(copy.masters[0]) }, false, worker.options.grpcDialOption, fileIds) return uploadError @@ -535,8 +546,12 @@ func detectMimeType(f *os.File) string { } func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, offset int64, tsNs int64) (chunk *filer_pb.FileChunk, err error) { + uploader, uploaderErr := operation.NewUploader() + if uploaderErr != nil { + return nil, fmt.Errorf("upload data: %v", uploaderErr) + } - finalFileId, uploadResult, flushErr, _ := operation.UploadWithRetry( + finalFileId, uploadResult, flushErr, _ := uploader.UploadWithRetry( worker, &filer_pb.AssignVolumeRequest{ Count: 1, diff --git a/weed/command/filer_meta_backup.go b/weed/command/filer_meta_backup.go index ff4a61e41..e8c4680ba 100644 --- a/weed/command/filer_meta_backup.go +++ b/weed/command/filer_meta_backup.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/viper" "google.golang.org/grpc" "reflect" + "strings" "time" "github.com/seaweedfs/seaweedfs/weed/pb" @@ -55,7 +56,7 @@ The backup writes to another filer store specified in a backup_filer.toml. func runFilerMetaBackup(cmd *Command, args []string) bool { - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() metaBackup.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client") // load backup_filer.toml @@ -63,9 +64,9 @@ func runFilerMetaBackup(cmd *Command, args []string) bool { v.SetConfigFile(*metaBackup.backupFilerConfig) if err := v.ReadInConfig(); err != nil { // Handle errors reading the config file - glog.Fatalf("Failed to load %s file.\nPlease use this command to generate the a %s.toml file\n"+ + glog.Fatalf("Failed to load %s file: %v\nPlease use this command to generate the a %s.toml file\n"+ " weed scaffold -config=%s -output=.\n\n\n", - *metaBackup.backupFilerConfig, "backup_filer", "filer") + *metaBackup.backupFilerConfig, err, "backup_filer", "filer") } if err := metaBackup.initStore(v); err != nil { @@ -197,17 +198,21 @@ func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error { metaBackup.clientEpoch++ + prefix := *metaBackup.filerDirectory + if !strings.HasSuffix(prefix, "/") { + prefix = prefix + "/" + } metadataFollowOption := &pb.MetadataFollowOption{ ClientName: "meta_backup", ClientId: metaBackup.clientId, ClientEpoch: metaBackup.clientEpoch, SelfSignature: 0, - PathPrefix: *metaBackup.filerDirectory, + PathPrefix: prefix, AdditionalPathPrefixes: nil, DirectoriesToWatch: nil, StartTsNs: startTime.UnixNano(), StopTsNs: 0, - EventErrorType: pb.TrivialOnError, + EventErrorType: pb.RetryForeverOnError, } return pb.FollowMetadata(pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, metadataFollowOption, processEventFnWithOffset) diff --git a/weed/command/filer_meta_tail.go b/weed/command/filer_meta_tail.go index 32855072b..d7a169535 100644 --- a/weed/command/filer_meta_tail.go +++ b/weed/command/filer_meta_tail.go @@ -45,7 +45,7 @@ var ( func runFilerMetaTail(cmd *Command, args []string) bool { - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") clientId := util.RandomInt32() diff --git a/weed/command/filer_remote_gateway.go b/weed/command/filer_remote_gateway.go index 61a5d26a2..78357cc04 100644 --- a/weed/command/filer_remote_gateway.go +++ b/weed/command/filer_remote_gateway.go @@ -78,7 +78,7 @@ var cmdFilerRemoteGateway = &Command{ func runFilerRemoteGateway(cmd *Command, args []string) bool { - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") remoteGatewayOptions.grpcDialOption = grpcDialOption diff --git a/weed/command/filer_remote_gateway_buckets.go b/weed/command/filer_remote_gateway_buckets.go index 9bb59dabb..f6fe9a99c 100644 --- a/weed/command/filer_remote_gateway_buckets.go +++ b/weed/command/filer_remote_gateway_buckets.go @@ -55,12 +55,12 @@ func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSo ClientId: option.clientId, ClientEpoch: option.clientEpoch, SelfSignature: 0, - PathPrefix: option.bucketsDir, + PathPrefix: option.bucketsDir + "/", AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote}, DirectoriesToWatch: nil, StartTsNs: lastOffsetTs.UnixNano(), StopTsNs: 0, - EventErrorType: pb.TrivialOnError, + EventErrorType: pb.RetryForeverOnError, } return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, metadataFollowOption, processEventFnWithOffset) diff --git a/weed/command/filer_remote_sync.go b/weed/command/filer_remote_sync.go index 2d6133367..77dd95134 100644 --- a/weed/command/filer_remote_sync.go +++ b/weed/command/filer_remote_sync.go @@ -73,7 +73,7 @@ var cmdFilerRemoteSynchronize = &Command{ func runFilerRemoteSynchronize(cmd *Command, args []string) bool { - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") remoteSyncOptions.grpcDialOption = grpcDialOption diff --git a/weed/command/filer_remote_sync_dir.go b/weed/command/filer_remote_sync_dir.go index 76f7e46d5..186523e45 100644 --- a/weed/command/filer_remote_sync_dir.go +++ b/weed/command/filer_remote_sync_dir.go @@ -64,17 +64,22 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour option.clientEpoch++ + prefix := mountedDir + if !strings.HasSuffix(prefix, "/") { + prefix = prefix + "/" + } + metadataFollowOption := &pb.MetadataFollowOption{ ClientName: "filer.remote.sync", ClientId: option.clientId, ClientEpoch: option.clientEpoch, SelfSignature: 0, - PathPrefix: mountedDir, + PathPrefix: prefix, AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote}, DirectoriesToWatch: nil, StartTsNs: lastOffsetTs.UnixNano(), StopTsNs: 0, - EventErrorType: pb.TrivialOnError, + EventErrorType: pb.RetryForeverOnError, } return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, metadataFollowOption, processEventFnWithOffset) diff --git a/weed/command/filer_replication.go b/weed/command/filer_replication.go index 4fca8158a..f53fdfb48 100644 --- a/weed/command/filer_replication.go +++ b/weed/command/filer_replication.go @@ -30,7 +30,7 @@ var cmdFilerReplicate = &Command{ func runFilerReplicate(cmd *Command, args []string) bool { - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() util.LoadConfiguration("replication", true) util.LoadConfiguration("notification", true) config := util.GetViper() diff --git a/weed/command/filer_sync.go b/weed/command/filer_sync.go index 9ad76e31b..90204af4a 100644 --- a/weed/command/filer_sync.go +++ b/weed/command/filer_sync.go @@ -118,7 +118,7 @@ var cmdFilerSynchronize = &Command{ func runFilerSynchronize(cmd *Command, args []string) bool { - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") grace.SetupProfiling(*syncCpuProfile, *syncMemProfile) @@ -296,12 +296,17 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, offsetTsNs) }) + prefix := sourcePath + if !strings.HasSuffix(prefix, "/") { + prefix = prefix + "/" + } + metadataFollowOption := &pb.MetadataFollowOption{ ClientName: clientName, ClientId: clientId, ClientEpoch: clientEpoch, SelfSignature: targetFilerSignature, - PathPrefix: sourcePath, + PathPrefix: prefix, AdditionalPathPrefixes: nil, DirectoriesToWatch: nil, StartTsNs: sourceFilerOffsetTsNs, @@ -469,14 +474,14 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str } } else { - // new key is outside of the watched directory + // new key is outside the watched directory if doDeleteFiles { key := buildKey(dataSink, message, targetPath, sourceOldKey, sourcePath) return dataSink.DeleteEntry(key, message.OldEntry.IsDirectory, message.DeleteChunks, message.Signatures) } } } else { - // old key is outside of the watched directory + // old key is outside the watched directory if strings.HasPrefix(string(sourceNewKey), sourcePath) { // new key is in the watched directory key := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath) @@ -486,7 +491,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str return nil } } else { - // new key is also outside of the watched directory + // new key is also outside the watched directory // skip } } diff --git a/weed/command/iam.go b/weed/command/iam.go index 95964994f..fa21803dd 100644 --- a/weed/command/iam.go +++ b/weed/command/iam.go @@ -47,7 +47,7 @@ func runIam(cmd *Command, args []string) bool { func (iamopt *IamOptions) startIamServer() bool { filerAddress := pb.ServerAddress(*iamopt.filer) - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") for { err := pb.WithGrpcFilerClient(false, 0, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { diff --git a/weed/command/master.go b/weed/command/master.go index f80d8faeb..914853d88 100644 --- a/weed/command/master.go +++ b/weed/command/master.go @@ -53,6 +53,7 @@ type MasterOptions struct { metricsIntervalSec *int raftResumeState *bool metricsHttpPort *int + metricsHttpIp *string heartbeatInterval *time.Duration electionTimeout *time.Duration raftHashicorp *bool @@ -77,6 +78,7 @@ func init() { m.metricsAddress = cmdMaster.Flag.String("metrics.address", "", "Prometheus gateway address <host>:<port>") m.metricsIntervalSec = cmdMaster.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds") m.metricsHttpPort = cmdMaster.Flag.Int("metricsPort", 0, "Prometheus metrics listen port") + m.metricsHttpIp = cmdMaster.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.") m.raftResumeState = cmdMaster.Flag.Bool("resumeState", false, "resume previous state on start master server") m.heartbeatInterval = cmdMaster.Flag.Duration("heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)") m.electionTimeout = cmdMaster.Flag.Duration("electionTimeout", 10*time.Second, "election timeout of master servers") @@ -103,7 +105,7 @@ var ( func runMaster(cmd *Command, args []string) bool { - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() util.LoadConfiguration("master", false) grace.SetupProfiling(*masterCpuProfile, *masterMemProfile) @@ -121,7 +123,15 @@ func runMaster(cmd *Command, args []string) bool { glog.Fatalf("volumeSizeLimitMB should be smaller than 30000") } - go stats_collect.StartMetricsServer(*m.ipBind, *m.metricsHttpPort) + switch { + case *m.metricsHttpIp != "": + // noting to do, use m.metricsHttpIp + case *m.ipBind != "": + *m.metricsHttpIp = *m.ipBind + case *m.ip != "": + *m.metricsHttpIp = *m.ip + } + go stats_collect.StartMetricsServer(*m.metricsHttpIp, *m.metricsHttpPort) startMaster(m, masterWhiteList) return true @@ -180,10 +190,10 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) { } } ms.SetRaftServer(raftServer) - r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET") - r.HandleFunc("/cluster/healthz", raftServer.HealthzHandler).Methods("GET", "HEAD") + r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods(http.MethodGet) + r.HandleFunc("/cluster/healthz", raftServer.HealthzHandler).Methods(http.MethodGet, http.MethodHead) if *masterOption.raftHashicorp { - r.HandleFunc("/raft/stats", raftServer.StatsRaftHandler).Methods("GET") + r.HandleFunc("/raft/stats", raftServer.StatsRaftHandler).Methods(http.MethodGet) } // starting grpc server grpcPort := *masterOption.portGrpc diff --git a/weed/command/master_follower.go b/weed/command/master_follower.go index 7217aff0b..504ddb6c3 100644 --- a/weed/command/master_follower.go +++ b/weed/command/master_follower.go @@ -68,7 +68,7 @@ var cmdMasterFollower = &Command{ func runMasterFollower(cmd *Command, args []string) bool { - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() util.LoadConfiguration("master", false) if *mf.portGrpc == 0 { diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index 742c38180..a5325b11e 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -66,7 +66,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool { // try to connect to filer filerAddresses := pb.ServerAddresses(*option.filer).ToAddresses() - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") var cipher bool var err error diff --git a/weed/command/mq_broker.go b/weed/command/mq_broker.go index e093ebc56..5eb304204 100644 --- a/weed/command/mq_broker.go +++ b/weed/command/mq_broker.go @@ -54,7 +54,7 @@ var cmdMqBroker = &Command{ func runMqBroker(cmd *Command, args []string) bool { - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() mqBrokerStandaloneOptions.masters = pb.ServerAddresses(*mqBrokerStandaloneOptions.mastersString).ToAddressMap() diff --git a/weed/command/s3.go b/weed/command/s3.go index b7bb2a546..e568de91b 100644 --- a/weed/command/s3.go +++ b/weed/command/s3.go @@ -49,6 +49,7 @@ type S3Options struct { tlsCACertificate *string tlsVerifyClientCert *bool metricsHttpPort *int + metricsHttpIp *string allowEmptyFolder *bool allowDeleteBucketNotEmpty *bool auditLogConfig *string @@ -75,6 +76,7 @@ func init() { s3StandaloneOptions.tlsCACertificate = cmdS3.Flag.String("cacert.file", "", "path to the TLS CA certificate file") s3StandaloneOptions.tlsVerifyClientCert = cmdS3.Flag.Bool("tlsVerifyClientCert", false, "whether to verify the client's certificate") s3StandaloneOptions.metricsHttpPort = cmdS3.Flag.Int("metricsPort", 0, "Prometheus metrics listen port") + s3StandaloneOptions.metricsHttpIp = cmdS3.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.") s3StandaloneOptions.allowEmptyFolder = cmdS3.Flag.Bool("allowEmptyFolder", true, "allow empty folders") s3StandaloneOptions.allowDeleteBucketNotEmpty = cmdS3.Flag.Bool("allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket") s3StandaloneOptions.localFilerSocket = cmdS3.Flag.String("localFilerSocket", "", "local filer socket path") @@ -163,17 +165,26 @@ var cmdS3 = &Command{ func runS3(cmd *Command, args []string) bool { - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() - go stats_collect.StartMetricsServer(*s3StandaloneOptions.bindIp, *s3StandaloneOptions.metricsHttpPort) + switch { + case *s3StandaloneOptions.metricsHttpIp != "": + // noting to do, use s3StandaloneOptions.metricsHttpIp + case *s3StandaloneOptions.bindIp != "": + *s3StandaloneOptions.metricsHttpIp = *s3StandaloneOptions.bindIp + } + go stats_collect.StartMetricsServer(*s3StandaloneOptions.metricsHttpIp, *s3StandaloneOptions.metricsHttpPort) return s3StandaloneOptions.startS3Server() } // GetCertificateWithUpdate Auto refreshing TSL certificate -func (S3opt *S3Options) GetCertificateWithUpdate(*tls.ClientHelloInfo) (*tls.Certificate, error) { - certs, err := S3opt.certProvider.KeyMaterial(context.Background()) +func (s3opt *S3Options) GetCertificateWithUpdate(*tls.ClientHelloInfo) (*tls.Certificate, error) { + certs, err := s3opt.certProvider.KeyMaterial(context.Background()) + if certs == nil { + return nil, err + } return &certs.Certs[0], err } @@ -320,6 +331,10 @@ func (s3opt *S3Options) startS3Server() bool { ClientAuth: clientAuth, ClientCAs: caCertPool, } + err = security.FixTlsConfig(util.GetViper(), httpS.TLSConfig) + if err != nil { + glog.Fatalf("error with tls config: %v", err) + } if *s3opt.portHttps == 0 { glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port) if s3ApiLocalListener != nil { diff --git a/weed/command/scaffold/filer.toml b/weed/command/scaffold/filer.toml index 574125207..728aecb53 100644 --- a/weed/command/scaffold/filer.toml +++ b/weed/command/scaffold/filer.toml @@ -285,7 +285,7 @@ password = "" ssl = false ssl_ca_file = "" ssl_cert_file = "" -ssl_key_file = " +ssl_key_file = "" insecure_skip_verify = false option_pool_size = 0 database = "seaweedfs" diff --git a/weed/command/scaffold/security.toml b/weed/command/scaffold/security.toml index c5b2a563c..113e5b016 100644 --- a/weed/command/scaffold/security.toml +++ b/weed/command/scaffold/security.toml @@ -95,18 +95,29 @@ allowed_commonNames = "" # comma-separated SSL certificate common names cert = "" key = "" -# volume server https options -# Note: work in progress! -# this does not work with other clients, e.g., "weed filer|mount" etc, yet. +# https client for master|volume|filer|etc connection +# It is necessary that the parameters [https.volume]|[https.master]|[https.filer] are set [https.client] enabled = true +cert = "" +key = "" +ca = "" +# volume server https options [https.volume] cert = "" key = "" ca = "" +# master server https options [https.master] cert = "" key = "" ca = "" + +# filer server https options +[https.filer] +cert = "" +key = "" +ca = "" +# disable_tls_verify_client_cert = true|false (default: false) diff --git a/weed/command/server.go b/weed/command/server.go index 503927629..ddcaf1f7e 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -66,6 +66,7 @@ var ( volumeMinFreeSpacePercent = cmdServer.Flag.String("volume.minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly (deprecated, use minFreeSpace instead).") volumeMinFreeSpace = cmdServer.Flag.String("volume.minFreeSpace", "", "min free disk space (value<=100 as percentage like 1, other as human readable bytes, like 10GiB). Low disk space will mark all volumes as ReadOnly.") serverMetricsHttpPort = cmdServer.Flag.Int("metricsPort", 0, "Prometheus metrics listen port") + serverMetricsHttpIp = cmdServer.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.") // pulseSeconds = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats") isStartingMasterServer = cmdServer.Flag.Bool("master", true, "whether to start master server") @@ -97,7 +98,7 @@ func init() { masterOptions.metricsIntervalSec = cmdServer.Flag.Int("master.metrics.intervalSeconds", 15, "Prometheus push interval in seconds") masterOptions.raftResumeState = cmdServer.Flag.Bool("master.resumeState", false, "resume previous state on start master server") masterOptions.raftHashicorp = cmdServer.Flag.Bool("master.raftHashicorp", false, "use hashicorp raft") - masterOptions.raftBootstrap = cmdMaster.Flag.Bool("master.raftBootstrap", false, "Whether to bootstrap the Raft cluster") + masterOptions.raftBootstrap = cmdServer.Flag.Bool("master.raftBootstrap", false, "Whether to bootstrap the Raft cluster") masterOptions.heartbeatInterval = cmdServer.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)") masterOptions.electionTimeout = cmdServer.Flag.Duration("master.electionTimeout", 10*time.Second, "election timeout of master servers") @@ -119,7 +120,6 @@ func init() { filerOptions.downloadMaxMBps = cmdServer.Flag.Int("filer.downloadMaxMBps", 0, "download max speed for each download request, in MB per second") filerOptions.diskType = cmdServer.Flag.String("filer.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag") filerOptions.exposeDirectoryData = cmdServer.Flag.Bool("filer.exposeDirectoryData", true, "expose directory data via filer. If false, filer UI will be innaccessible.") - filerOptions.joinExistingFiler = cmdServer.Flag.Bool("filer.joinExistingFiler", false, "enable if new filer wants to join existing cluster") serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port") serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 0, "volume server grpc listen port") @@ -179,7 +179,7 @@ func runServer(cmd *Command, args []string) bool { go http.ListenAndServe(fmt.Sprintf(":%d", *serverOptions.debugPort), nil) } - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() util.LoadConfiguration("master", false) grace.SetupProfiling(*serverOptions.cpuprofile, *serverOptions.memprofile) @@ -207,6 +207,10 @@ func runServer(cmd *Command, args []string) bool { serverBindIp = serverIp } + if *serverMetricsHttpIp == "" { + *serverMetricsHttpIp = *serverBindIp + } + // ip address masterOptions.ip = serverIp masterOptions.ipBind = serverBindIp @@ -245,7 +249,7 @@ func runServer(cmd *Command, args []string) bool { webdavOptions.filer = &filerAddress mqBrokerOptions.filerGroup = filerOptions.filerGroup - go stats_collect.StartMetricsServer(*serverBindIp, *serverMetricsHttpPort) + go stats_collect.StartMetricsServer(*serverMetricsHttpIp, *serverMetricsHttpPort) folders := strings.Split(*volumeDataFolders, ",") diff --git a/weed/command/shell.go b/weed/command/shell.go index f78ba89fc..1e921411b 100644 --- a/weed/command/shell.go +++ b/weed/command/shell.go @@ -35,7 +35,7 @@ var cmdShell = &Command{ func runShell(command *Command, args []string) bool { - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() shellOptions.GrpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client") shellOptions.Directory = "/" diff --git a/weed/command/update.go b/weed/command/update.go index 314a903f2..bf871d654 100644 --- a/weed/command/update.go +++ b/weed/command/update.go @@ -20,6 +20,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/util" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" "golang.org/x/net/context/ctxhttp" ) @@ -198,7 +199,7 @@ func GitHubLatestRelease(ctx context.Context, ver string, owner, repo string) (R if err != nil { return Release{}, err } - defer util.CloseResponse(res) + defer util_http.CloseResponse(res) if res.StatusCode != http.StatusOK { content := res.Header.Get("Content-Type") @@ -258,7 +259,7 @@ func getGithubData(ctx context.Context, url string) ([]byte, error) { if err != nil { return nil, err } - defer util.CloseResponse(res) + defer util_http.CloseResponse(res) if res.StatusCode != http.StatusOK { return nil, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status) @@ -308,7 +309,12 @@ func extractToFile(buf []byte, filename, target string) error { trd := tar.NewReader(gr) hdr, terr := trd.Next() if terr != nil { - glog.Errorf("uncompress file(%s) failed:%s", hdr.Name, terr) + if hdr != nil { + glog.Errorf("uncompress file(%s) failed:%s", hdr.Name, terr) + } else { + glog.Errorf("uncompress file is nil, failed:%s", terr) + } + return terr } rd = trd diff --git a/weed/command/upload.go b/weed/command/upload.go index 3e6b8f9a2..7135a707a 100644 --- a/weed/command/upload.go +++ b/weed/command/upload.go @@ -69,7 +69,7 @@ var cmdUpload = &Command{ func runUpload(cmd *Command, args []string) bool { - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client") defaultReplication, err := readMasterConfiguration(grpcDialOption, pb.ServerAddress(*upload.master)) diff --git a/weed/command/volume.go b/weed/command/volume.go index 852989d1f..1078d8d6c 100644 --- a/weed/command/volume.go +++ b/weed/command/volume.go @@ -65,6 +65,7 @@ type VolumeServerOptions struct { pprof *bool preStopSeconds *int metricsHttpPort *int + metricsHttpIp *string // pulseSeconds *int inflightUploadDataTimeout *time.Duration hasSlowRead *bool @@ -99,6 +100,7 @@ func init() { v.concurrentDownloadLimitMB = cmdVolume.Flag.Int("concurrentDownloadLimitMB", 256, "limit total concurrent download size") v.pprof = cmdVolume.Flag.Bool("pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile") v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port") + v.metricsHttpIp = cmdVolume.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.") v.idxFolder = cmdVolume.Flag.String("dir.idx", "", "directory to store .idx files") v.inflightUploadDataTimeout = cmdVolume.Flag.Duration("inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout of volume servers") v.hasSlowRead = cmdVolume.Flag.Bool("hasSlowRead", true, "<experimental> if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.") @@ -123,7 +125,7 @@ var ( func runVolume(cmd *Command, args []string) bool { - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() // If --pprof is set we assume the caller wants to be able to collect // cpu and memory profiles via go tool pprof @@ -131,7 +133,15 @@ func runVolume(cmd *Command, args []string) bool { grace.SetupProfiling(*v.cpuProfile, *v.memProfile) } - go stats_collect.StartMetricsServer(*v.bindIp, *v.metricsHttpPort) + switch { + case *v.metricsHttpIp != "": + // noting to do, use v.metricsHttpIp + case *v.bindIp != "": + *v.metricsHttpIp = *v.bindIp + case *v.ip != "": + *v.metricsHttpIp = *v.ip + } + go stats_collect.StartMetricsServer(*v.metricsHttpIp, *v.metricsHttpPort) minFreeSpaces := util.MustParseMinFreeSpace(*minFreeSpace, *minFreeSpacePercent) v.masters = pb.ServerAddresses(*v.mastersString).ToAddresses() diff --git a/weed/command/webdav.go b/weed/command/webdav.go index f0e738f4a..1d1a43eda 100644 --- a/weed/command/webdav.go +++ b/weed/command/webdav.go @@ -60,7 +60,7 @@ var cmdWebDav = &Command{ func runWebDav(cmd *Command, args []string) bool { - util.LoadConfiguration("security", false) + util.LoadSecurityConfiguration() glog.V(0).Infof("Starting Seaweed WebDav Server %s at https port %d", util.Version(), *webDavStandaloneOptions.port) diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go index ee2afa30f..1d175651d 100644 --- a/weed/filer/abstract_sql/abstract_sql_store.go +++ b/weed/filer/abstract_sql/abstract_sql_store.go @@ -7,6 +7,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket" "github.com/seaweedfs/seaweedfs/weed/util" "strings" "sync" @@ -140,6 +141,8 @@ func (store *AbstractSqlStore) getTxOrDB(ctx context.Context, fullpath util.Full } } + } else { + err = fmt.Errorf("invalid bucket name %s", bucket) } return @@ -340,6 +343,9 @@ func (store *AbstractSqlStore) Shutdown() { } func isValidBucket(bucket string) bool { + if s3bucket.VerifyS3BucketName(bucket) != nil { + return false + } return bucket != DEFAULT_TABLE && bucket != "" } diff --git a/weed/filer/filechunk_group.go b/weed/filer/filechunk_group.go index c89527710..dbd08b42b 100644 --- a/weed/filer/filechunk_group.go +++ b/weed/filer/filechunk_group.go @@ -1,10 +1,11 @@ package filer import ( + "sync" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/util/chunk_cache" "github.com/seaweedfs/seaweedfs/weed/wdclient" - "sync" ) type ChunkGroup struct { @@ -54,9 +55,11 @@ func (group *ChunkGroup) ReadDataAt(fileSize int64, buff []byte, offset int64) ( section, found := group.sections[si] rangeStart, rangeStop := max(offset, int64(si*SectionSize)), min(offset+int64(len(buff)), int64((si+1)*SectionSize)) if !found { + rangeStop = min(rangeStop, fileSize) for i := rangeStart; i < rangeStop; i++ { buff[i-offset] = 0 } + n = int(int64(n) + rangeStop - rangeStart) continue } xn, xTsNs, xErr := section.readDataAt(group, fileSize, buff[rangeStart-offset:rangeStop-offset], rangeStart) diff --git a/weed/filer/filechunk_manifest.go b/weed/filer/filechunk_manifest.go index 7ea2f0353..e9ae1800c 100644 --- a/weed/filer/filechunk_manifest.go +++ b/weed/filer/filechunk_manifest.go @@ -15,6 +15,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/util" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) const ( @@ -120,7 +121,7 @@ func fetchChunkRange(buffer []byte, lookupFileIdFn wdclient.LookupFileIdFunction glog.Errorf("operation LookupFileId %s failed, err: %v", fileId, err) return 0, err } - return util.RetriedFetchChunkData(buffer, urlStrings, cipherKey, isGzipped, false, offset) + return util_http.RetriedFetchChunkData(buffer, urlStrings, cipherKey, isGzipped, false, offset) } func retriedStreamFetchChunkData(writer io.Writer, urlStrings []string, jwt string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) (err error) { @@ -132,7 +133,7 @@ func retriedStreamFetchChunkData(writer io.Writer, urlStrings []string, jwt stri for _, urlString := range urlStrings { var localProcessed int var writeErr error - shouldRetry, err = util.ReadUrlAsStreamAuthenticated(urlString+"?readDeleted=true", jwt, cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) { + shouldRetry, err = util_http.ReadUrlAsStreamAuthenticated(urlString+"?readDeleted=true", jwt, cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) { if totalWritten > localProcessed { toBeSkipped := totalWritten - localProcessed if len(data) <= toBeSkipped { diff --git a/weed/filer/filer.go b/weed/filer/filer.go index 016bfc8fa..80be0b88e 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -78,7 +78,7 @@ func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerH return f } -func (f *Filer) MaybeBootstrapFromPeers(self pb.ServerAddress, existingNodes []*master_pb.ClusterNodeUpdate, snapshotTime time.Time) (err error) { +func (f *Filer) MaybeBootstrapFromOnePeer(self pb.ServerAddress, existingNodes []*master_pb.ClusterNodeUpdate, snapshotTime time.Time) (err error) { if len(existingNodes) == 0 { return } @@ -91,25 +91,13 @@ func (f *Filer) MaybeBootstrapFromPeers(self pb.ServerAddress, existingNodes []* } glog.V(0).Infof("bootstrap from %v clientId:%d", earliestNode.Address, f.UniqueFilerId) - f.UniqueFilerEpoch++ - - metadataFollowOption := &pb.MetadataFollowOption{ - ClientName: "bootstrap", - ClientId: f.UniqueFilerId, - ClientEpoch: f.UniqueFilerEpoch, - SelfSignature: f.Signature, - PathPrefix: "/", - AdditionalPathPrefixes: nil, - DirectoriesToWatch: nil, - StartTsNs: 0, - StopTsNs: snapshotTime.UnixNano(), - EventErrorType: pb.FatalOnError, - } - err = pb.FollowMetadata(pb.ServerAddress(earliestNode.Address), f.GrpcDialOption, metadataFollowOption, func(resp *filer_pb.SubscribeMetadataResponse) error { - return Replay(f.Store, resp) + return pb.WithFilerClient(false, f.UniqueFilerId, pb.ServerAddress(earliestNode.Address), f.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { + return filer_pb.StreamBfs(client, "/", snapshotTime.UnixNano(), func(parentPath util.FullPath, entry *filer_pb.Entry) error { + return f.Store.InsertEntry(context.Background(), FromPbEntry(string(parentPath), entry)) + }) }) - return + } func (f *Filer) AggregateFromPeers(self pb.ServerAddress, existingNodes []*master_pb.ClusterNodeUpdate, startFrom time.Time) { diff --git a/weed/filer/filer_conf.go b/weed/filer/filer_conf.go index 25bf83771..69755a16a 100644 --- a/weed/filer/filer_conf.go +++ b/weed/filer/filer_conf.go @@ -102,7 +102,7 @@ func (fc *FilerConf) LoadFromBytes(data []byte) (err error) { func (fc *FilerConf) doLoadConf(conf *filer_pb.FilerConf) (err error) { for _, location := range conf.Locations { - err = fc.AddLocationConf(location) + err = fc.SetLocationConf(location) if err != nil { // this is not recoverable return nil @@ -111,7 +111,24 @@ func (fc *FilerConf) doLoadConf(conf *filer_pb.FilerConf) (err error) { return nil } +func (fc *FilerConf) GetLocationConf(locationPrefix string)(locConf *filer_pb.FilerConf_PathConf, found bool) { + return fc.rules.Get([]byte(locationPrefix)) +} + +func (fc *FilerConf) SetLocationConf(locConf *filer_pb.FilerConf_PathConf) (err error) { + err = fc.rules.Put([]byte(locConf.LocationPrefix), locConf) + if err != nil { + glog.Errorf("put location prefix: %v", err) + } + return +} + func (fc *FilerConf) AddLocationConf(locConf *filer_pb.FilerConf_PathConf) (err error) { + existingConf, found := fc.rules.Get([]byte(locConf.LocationPrefix)) + if found { + mergePathConf(existingConf, locConf) + locConf = existingConf + } err = fc.rules.Put([]byte(locConf.LocationPrefix), locConf) if err != nil { glog.Errorf("put location prefix: %v", err) @@ -170,6 +187,7 @@ func mergePathConf(a, b *filer_pb.FilerConf_PathConf) { a.DataCenter = util.Nvl(b.DataCenter, a.DataCenter) a.Rack = util.Nvl(b.Rack, a.Rack) a.DataNode = util.Nvl(b.DataNode, a.DataNode) + a.DisableChunkDeletion = b.DisableChunkDeletion || a.DisableChunkDeletion } func (fc *FilerConf) ToProto() *filer_pb.FilerConf { diff --git a/weed/filer/filer_delete_entry.go b/weed/filer/filer_delete_entry.go index 64546940e..0ae421981 100644 --- a/weed/filer/filer_delete_entry.go +++ b/weed/filer/filer_delete_entry.go @@ -17,7 +17,7 @@ const ( type OnChunksFunc func([]*filer_pb.FileChunk) error type OnHardLinkIdsFunc func([]HardLinkId) error -func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isFromOtherCluster bool, signatures []int32) (err error) { +func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isFromOtherCluster bool, signatures []int32, ifNotModifiedAfter int64) (err error) { if p == "/" { return nil } @@ -26,6 +26,9 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR if findErr != nil { return findErr } + if ifNotModifiedAfter > 0 && entry.Attr.Mtime.Unix() > ifNotModifiedAfter { + return nil + } isDeleteCollection := f.isBucket(entry) if entry.IsDirectory() { // delete the folder children, not including the folder itself @@ -50,12 +53,12 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR } if shouldDeleteChunks && !isDeleteCollection { - f.DirectDeleteChunks(entry.GetChunks()) + f.DeleteChunks(p, entry.GetChunks()) } if isDeleteCollection { collectionName := entry.Name() - f.doDeleteCollection(collectionName) + f.DoDeleteCollection(collectionName) } return nil @@ -133,7 +136,7 @@ func (f *Filer) doDeleteEntryMetaAndData(ctx context.Context, entry *Entry, shou return nil } -func (f *Filer) doDeleteCollection(collectionName string) (err error) { +func (f *Filer) DoDeleteCollection(collectionName string) (err error) { return f.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error { _, err := client.CollectionDelete(context.Background(), &master_pb.CollectionDeleteRequest{ diff --git a/weed/filer/filer_deletion.go b/weed/filer/filer_deletion.go index 84c5ed4c0..362c7c51b 100644 --- a/weed/filer/filer_deletion.go +++ b/weed/filer/filer_deletion.go @@ -1,11 +1,12 @@ package filer import ( - "github.com/seaweedfs/seaweedfs/weed/storage" - "github.com/seaweedfs/seaweedfs/weed/util" "strings" "time" + "github.com/seaweedfs/seaweedfs/weed/storage" + "github.com/seaweedfs/seaweedfs/weed/util" + "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" @@ -22,6 +23,7 @@ func LookupByMasterClientFn(masterClient *wdclient.MasterClient) func(vids []str locations = append(locations, operation.Location{ Url: loc.Url, PublicUrl: loc.PublicUrl, + GrpcPort: loc.GrpcPort, }) } m[vid] = &operation.LookupResult{ @@ -53,7 +55,7 @@ func (f *Filer) loopProcessingDeletion() { fileIds = fileIds[:0] } deletionCount = len(toDeleteFileIds) - _, err := operation.DeleteFilesWithLookupVolumeId(f.GrpcDialOption, toDeleteFileIds, lookupFunc) + _, err := operation.DeleteFileIdsWithLookupVolumeId(f.GrpcDialOption, toDeleteFileIds, lookupFunc) if err != nil { if !strings.Contains(err.Error(), storage.ErrorDeleted.Error()) { glog.V(0).Infof("deleting fileIds len=%d error: %v", deletionCount, err) @@ -70,50 +72,6 @@ func (f *Filer) loopProcessingDeletion() { } } -func (f *Filer) doDeleteFileIds(fileIds []string) { - - lookupFunc := LookupByMasterClientFn(f.MasterClient) - DeletionBatchSize := 100000 // roughly 20 bytes cost per file id. - - for len(fileIds) > 0 { - var toDeleteFileIds []string - if len(fileIds) > DeletionBatchSize { - toDeleteFileIds = fileIds[:DeletionBatchSize] - fileIds = fileIds[DeletionBatchSize:] - } else { - toDeleteFileIds = fileIds - fileIds = fileIds[:0] - } - deletionCount := len(toDeleteFileIds) - _, err := operation.DeleteFilesWithLookupVolumeId(f.GrpcDialOption, toDeleteFileIds, lookupFunc) - if err != nil { - if !strings.Contains(err.Error(), storage.ErrorDeleted.Error()) { - glog.V(0).Infof("deleting fileIds len=%d error: %v", deletionCount, err) - } - } - } -} - -func (f *Filer) DirectDeleteChunks(chunks []*filer_pb.FileChunk) { - var fileIdsToDelete []string - for _, chunk := range chunks { - if !chunk.IsChunkManifest { - fileIdsToDelete = append(fileIdsToDelete, chunk.GetFileIdString()) - continue - } - dataChunks, manifestResolveErr := ResolveOneChunkManifest(f.MasterClient.LookupFileId, chunk) - if manifestResolveErr != nil { - glog.V(0).Infof("failed to resolve manifest %s: %v", chunk.FileId, manifestResolveErr) - } - for _, dChunk := range dataChunks { - fileIdsToDelete = append(fileIdsToDelete, dChunk.GetFileIdString()) - } - fileIdsToDelete = append(fileIdsToDelete, chunk.GetFileIdString()) - } - - f.doDeleteFileIds(fileIdsToDelete) -} - func (f *Filer) DeleteUncommittedChunks(chunks []*filer_pb.FileChunk) { f.doDeleteChunks(chunks) } diff --git a/weed/filer/filer_notify.go b/weed/filer/filer_notify.go index db953d398..621d4c227 100644 --- a/weed/filer/filer_notify.go +++ b/weed/filer/filer_notify.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/seaweedfs/seaweedfs/weed/util/log_buffer" "io" - "math" "regexp" "strings" "time" @@ -116,101 +115,34 @@ var ( func (f *Filer) ReadPersistedLogBuffer(startPosition log_buffer.MessagePosition, stopTsNs int64, eachLogEntryFn log_buffer.EachLogEntryFuncType) (lastTsNs int64, isDone bool, err error) { - startDate := fmt.Sprintf("%04d-%02d-%02d", startPosition.Year(), startPosition.Month(), startPosition.Day()) - startHourMinute := fmt.Sprintf("%02d-%02d", startPosition.Hour(), startPosition.Minute()) - var stopDate, stopHourMinute string - if stopTsNs != 0 { - stopTime := time.Unix(0, stopTsNs+24*60*60*int64(time.Nanosecond)).UTC() - stopDate = fmt.Sprintf("%04d-%02d-%02d", stopTime.Year(), stopTime.Month(), stopTime.Day()) - stopHourMinute = fmt.Sprintf("%02d-%02d", stopTime.Hour(), stopTime.Minute()) - } - - sizeBuf := make([]byte, 4) - startTsNs := startPosition.UnixNano() - - dayEntries, _, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, math.MaxInt32, "", "", "") - if listDayErr != nil { - return lastTsNs, isDone, fmt.Errorf("fail to list log by day: %v", listDayErr) + visitor, visitErr := f.collectPersistedLogBuffer(startPosition, stopTsNs) + if visitErr != nil { + if visitErr == io.EOF { + return + } + err = fmt.Errorf("reading from persisted logs: %v", visitErr) + return } - for _, dayEntry := range dayEntries { - if stopDate != "" { - if strings.Compare(dayEntry.Name(), stopDate) > 0 { + var logEntry *filer_pb.LogEntry + for { + logEntry, visitErr = visitor.GetNext() + if visitErr != nil { + if visitErr == io.EOF { break } + err = fmt.Errorf("read next from persisted logs: %v", visitErr) + return } - // println("checking day", dayEntry.FullPath) - hourMinuteEntries, _, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, math.MaxInt32, "", "", "") - if listHourMinuteErr != nil { - return lastTsNs, isDone, fmt.Errorf("fail to list log %s by day: %v", dayEntry.Name(), listHourMinuteErr) + isDone, visitErr = eachLogEntryFn(logEntry) + if visitErr != nil { + err = fmt.Errorf("process persisted log entry: %v", visitErr) + return } - for _, hourMinuteEntry := range hourMinuteEntries { - // println("checking hh-mm", hourMinuteEntry.FullPath) - if dayEntry.Name() == startDate { - hourMinute := util.FileNameBase(hourMinuteEntry.Name()) - if strings.Compare(hourMinute, startHourMinute) < 0 { - continue - } - } - if dayEntry.Name() == stopDate { - hourMinute := util.FileNameBase(hourMinuteEntry.Name()) - if strings.Compare(hourMinute, stopHourMinute) > 0 { - break - } - } - // println("processing", hourMinuteEntry.FullPath) - chunkedFileReader := NewChunkStreamReaderFromFiler(f.MasterClient, hourMinuteEntry.GetChunks()) - if lastTsNs, err = ReadEachLogEntry(chunkedFileReader, sizeBuf, startTsNs, stopTsNs, eachLogEntryFn); err != nil { - chunkedFileReader.Close() - if err == io.EOF { - continue - } - if VolumeNotFoundPattern.MatchString(err.Error()) { - glog.Warningf("skipping reading %s: %v", hourMinuteEntry.FullPath, err) - continue - } - return lastTsNs, isDone, fmt.Errorf("reading %s: %v", hourMinuteEntry.FullPath, err) - } - chunkedFileReader.Close() + lastTsNs = logEntry.TsNs + if isDone { + return } } - return lastTsNs, isDone, nil -} - -func ReadEachLogEntry(r io.Reader, sizeBuf []byte, startTsNs, stopTsNs int64, eachLogEntryFn log_buffer.EachLogEntryFuncType) (lastTsNs int64, err error) { - for { - n, err := r.Read(sizeBuf) - if err != nil { - return lastTsNs, err - } - if n != 4 { - return lastTsNs, fmt.Errorf("size %d bytes, expected 4 bytes", n) - } - size := util.BytesToUint32(sizeBuf) - // println("entry size", size) - entryData := make([]byte, size) - n, err = r.Read(entryData) - if err != nil { - return lastTsNs, err - } - if n != int(size) { - return lastTsNs, fmt.Errorf("entry data %d bytes, expected %d bytes", n, size) - } - logEntry := &filer_pb.LogEntry{} - if err = proto.Unmarshal(entryData, logEntry); err != nil { - return lastTsNs, err - } - if logEntry.TsNs <= startTsNs { - continue - } - if stopTsNs != 0 && logEntry.TsNs > stopTsNs { - return lastTsNs, err - } - // println("each log: ", logEntry.TsNs) - if _, err := eachLogEntryFn(logEntry); err != nil { - return lastTsNs, err - } else { - lastTsNs = logEntry.TsNs - } - } + return } diff --git a/weed/filer/filer_notify_append.go b/weed/filer/filer_notify_append.go index 66ce24871..3c9a3496c 100644 --- a/weed/filer/filer_notify_append.go +++ b/weed/filer/filer_notify_append.go @@ -77,7 +77,13 @@ func (f *Filer) assignAndUpload(targetFile string, data []byte) (*operation.Assi PairMap: nil, Jwt: assignResult.Auth, } - uploadResult, err := operation.UploadData(data, uploadOption) + + uploader, err := operation.NewUploader() + if err != nil { + return nil, nil, fmt.Errorf("upload data %s: %v", targetUrl, err) + } + + uploadResult, err := uploader.UploadData(data, uploadOption) if err != nil { return nil, nil, fmt.Errorf("upload data %s: %v", targetUrl, err) } diff --git a/weed/filer/filer_notify_read.go b/weed/filer/filer_notify_read.go new file mode 100644 index 000000000..115a925e9 --- /dev/null +++ b/weed/filer/filer_notify_read.go @@ -0,0 +1,352 @@ +package filer + +import ( + "container/heap" + "context" + "fmt" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/util/log_buffer" + "github.com/seaweedfs/seaweedfs/weed/wdclient" + "google.golang.org/protobuf/proto" + "io" + "math" + "strings" + "time" + + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +type LogFileEntry struct { + TsNs int64 + FileEntry *Entry +} + +func (f *Filer) collectPersistedLogBuffer(startPosition log_buffer.MessagePosition, stopTsNs int64) (v *OrderedLogVisitor, err error) { + + if stopTsNs != 0 && startPosition.Time.UnixNano() > stopTsNs { + return nil, io.EOF + } + + startDate := fmt.Sprintf("%04d-%02d-%02d", startPosition.Year(), startPosition.Month(), startPosition.Day()) + + dayEntries, _, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, math.MaxInt32, "", "", "") + if listDayErr != nil { + return nil, fmt.Errorf("fail to list log by day: %v", listDayErr) + } + + return NewOrderedLogVisitor(f, startPosition, stopTsNs, dayEntries) + +} + +// ---------- +type LogEntryItem struct { + Entry *filer_pb.LogEntry + filer string +} + +// LogEntryItemPriorityQueue a priority queue for LogEntry +type LogEntryItemPriorityQueue []*LogEntryItem + +func (pq LogEntryItemPriorityQueue) Len() int { return len(pq) } +func (pq LogEntryItemPriorityQueue) Less(i, j int) bool { + return pq[i].Entry.TsNs < pq[j].Entry.TsNs +} +func (pq LogEntryItemPriorityQueue) Swap(i, j int) { pq[i], pq[j] = pq[j], pq[i] } +func (pq *LogEntryItemPriorityQueue) Push(x any) { + item := x.(*LogEntryItem) + *pq = append(*pq, item) +} +func (pq *LogEntryItemPriorityQueue) Pop() any { + n := len(*pq) + item := (*pq)[n-1] + *pq = (*pq)[:n-1] + return item +} + +// ---------- + +type OrderedLogVisitor struct { + perFilerIteratorMap map[string]*LogFileQueueIterator + pq *LogEntryItemPriorityQueue + logFileEntryCollector *LogFileEntryCollector +} + +func NewOrderedLogVisitor(f *Filer, startPosition log_buffer.MessagePosition, stopTsNs int64, dayEntries []*Entry) (*OrderedLogVisitor, error) { + + perFilerQueueMap := make(map[string]*LogFileQueueIterator) + // initialize the priority queue + pq := &LogEntryItemPriorityQueue{} + heap.Init(pq) + + t := &OrderedLogVisitor{ + perFilerIteratorMap: perFilerQueueMap, + pq: pq, + logFileEntryCollector: NewLogFileEntryCollector(f, startPosition, stopTsNs, dayEntries), + } + if err := t.logFileEntryCollector.collectMore(t); err != nil && err != io.EOF { + return nil, err + } + return t, nil +} + +func (o *OrderedLogVisitor) GetNext() (logEntry *filer_pb.LogEntry, err error) { + if o.pq.Len() == 0 { + return nil, io.EOF + } + item := heap.Pop(o.pq).(*LogEntryItem) + filerId := item.filer + + // fill the pq with the next log entry from the same filer + it := o.perFilerIteratorMap[filerId] + next, nextErr := it.getNext(o) + if nextErr != nil { + if nextErr == io.EOF { + // do nothing since the filer has no more log entries + }else { + return nil, fmt.Errorf("failed to get next log entry: %v", nextErr) + } + } else { + heap.Push(o.pq, &LogEntryItem{ + Entry: next, + filer: filerId, + }) + } + return item.Entry, nil +} + +func getFilerId(name string) string { + idx := strings.LastIndex(name, ".") + if idx < 0 { + return "" + } + return name[idx+1:] +} + +// ---------- + +type LogFileEntryCollector struct { + f *Filer + startTsNs int64 + stopTsNs int64 + dayEntryQueue *util.Queue[*Entry] + startDate string + startHourMinute string + stopDate string + stopHourMinute string +} + +func NewLogFileEntryCollector(f *Filer, startPosition log_buffer.MessagePosition, stopTsNs int64, dayEntries []*Entry) *LogFileEntryCollector { + dayEntryQueue := util.NewQueue[*Entry]() + for _, dayEntry := range dayEntries { + dayEntryQueue.Enqueue(dayEntry) + // println("enqueue day entry", dayEntry.Name()) + } + + startDate := fmt.Sprintf("%04d-%02d-%02d", startPosition.Year(), startPosition.Month(), startPosition.Day()) + startHourMinute := fmt.Sprintf("%02d-%02d", startPosition.Hour(), startPosition.Minute()) + var stopDate, stopHourMinute string + if stopTsNs != 0 { + stopTime := time.Unix(0, stopTsNs+24*60*60*int64(time.Nanosecond)).UTC() + stopDate = fmt.Sprintf("%04d-%02d-%02d", stopTime.Year(), stopTime.Month(), stopTime.Day()) + stopHourMinute = fmt.Sprintf("%02d-%02d", stopTime.Hour(), stopTime.Minute()) + } + + return &LogFileEntryCollector{ + f: f, + startTsNs: startPosition.UnixNano(), + stopTsNs: stopTsNs, + dayEntryQueue: dayEntryQueue, + startDate: startDate, + startHourMinute: startHourMinute, + stopDate: stopDate, + stopHourMinute: stopHourMinute, + } +} + +func (c *LogFileEntryCollector) hasMore() bool { + return c.dayEntryQueue.Len() > 0 +} + +func (c *LogFileEntryCollector) collectMore(v *OrderedLogVisitor) (err error) { + dayEntry := c.dayEntryQueue.Dequeue() + if dayEntry == nil { + return io.EOF + } + // println("dequeue day entry", dayEntry.Name()) + if c.stopDate != "" { + if strings.Compare(dayEntry.Name(), c.stopDate) > 0 { + return io.EOF + } + } + + hourMinuteEntries, _, listHourMinuteErr := c.f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, math.MaxInt32, "", "", "") + if listHourMinuteErr != nil { + return fmt.Errorf("fail to list log %s by day: %v", dayEntry.Name(), listHourMinuteErr) + } + freshFilerIds := make(map[string]string) + for _, hourMinuteEntry := range hourMinuteEntries { + // println("checking hh-mm", hourMinuteEntry.FullPath) + hourMinute := util.FileNameBase(hourMinuteEntry.Name()) + if dayEntry.Name() == c.startDate { + if strings.Compare(hourMinute, c.startHourMinute) < 0 { + continue + } + } + if dayEntry.Name() == c.stopDate { + if strings.Compare(hourMinute, c.stopHourMinute) > 0 { + break + } + } + + tsMinute := fmt.Sprintf("%s-%s", dayEntry.Name(), hourMinute) + // println(" enqueue", tsMinute) + t, parseErr := time.Parse("2006-01-02-15-04", tsMinute) + if parseErr != nil { + glog.Errorf("failed to parse %s: %v", tsMinute, parseErr) + continue + } + filerId := getFilerId(hourMinuteEntry.Name()) + iter, found := v.perFilerIteratorMap[filerId] + if !found { + iter = newLogFileQueueIterator(c.f.MasterClient, util.NewQueue[*LogFileEntry](), c.startTsNs, c.stopTsNs) + v.perFilerIteratorMap[filerId] = iter + freshFilerIds[filerId] = hourMinuteEntry.Name() + } + iter.q.Enqueue(&LogFileEntry{ + TsNs: t.UnixNano(), + FileEntry: hourMinuteEntry, + }) + } + + // fill the pq with the next log entry if it is a new filer + for filerId, entryName := range freshFilerIds { + iter, found := v.perFilerIteratorMap[filerId] + if !found { + glog.Errorf("Unexpected! failed to find iterator for filer %s", filerId) + continue + } + next, nextErr := iter.getNext(v) + if nextErr != nil { + if nextErr == io.EOF { + // do nothing since the filer has no more log entries + }else { + return fmt.Errorf("failed to get next log entry for %v: %v", entryName, err) + } + } else { + heap.Push(v.pq, &LogEntryItem{ + Entry: next, + filer: filerId, + }) + } + } + + return nil +} + +// ---------- + +type LogFileQueueIterator struct { + q *util.Queue[*LogFileEntry] + masterClient *wdclient.MasterClient + startTsNs int64 + stopTsNs int64 + currentFileIterator *LogFileIterator +} + +func newLogFileQueueIterator(masterClient *wdclient.MasterClient, q *util.Queue[*LogFileEntry], startTsNs, stopTsNs int64) *LogFileQueueIterator { + return &LogFileQueueIterator{ + q: q, + masterClient: masterClient, + startTsNs: startTsNs, + stopTsNs: stopTsNs, + } +} + +// getNext will return io.EOF when done +func (iter *LogFileQueueIterator) getNext(v *OrderedLogVisitor) (logEntry *filer_pb.LogEntry, err error) { + for { + if iter.currentFileIterator != nil { + logEntry, err = iter.currentFileIterator.getNext() + if err != io.EOF { + return + } + } + // now either iter.currentFileIterator is nil or err is io.EOF + if iter.q.Len() == 0 { + return nil, io.EOF + } + t := iter.q.Dequeue() + if t == nil { + continue + } + // skip the file if it is after the stopTsNs + if iter.stopTsNs != 0 && t.TsNs > iter.stopTsNs { + return nil, io.EOF + } + next := iter.q.Peek() + if next == nil { + if collectErr := v.logFileEntryCollector.collectMore(v); collectErr != nil && collectErr != io.EOF { + return nil, collectErr + } + } + // skip the file if the next entry is before the startTsNs + if next != nil && next.TsNs <= iter.startTsNs { + continue + } + iter.currentFileIterator = newLogFileIterator(iter.masterClient, t.FileEntry, iter.startTsNs, iter.stopTsNs) + } +} + +// ---------- + +type LogFileIterator struct { + r io.Reader + sizeBuf []byte + startTsNs int64 + stopTsNs int64 +} + +func newLogFileIterator(masterClient *wdclient.MasterClient, fileEntry *Entry, startTsNs, stopTsNs int64) *LogFileIterator { + return &LogFileIterator{ + r: NewChunkStreamReaderFromFiler(masterClient, fileEntry.Chunks), + sizeBuf: make([]byte, 4), + startTsNs: startTsNs, + stopTsNs: stopTsNs, + } +} + +// getNext will return io.EOF when done +func (iter *LogFileIterator) getNext() (logEntry *filer_pb.LogEntry, err error) { + var n int + for { + n, err = iter.r.Read(iter.sizeBuf) + if err != nil { + return + } + if n != 4 { + return nil, fmt.Errorf("size %d bytes, expected 4 bytes", n) + } + size := util.BytesToUint32(iter.sizeBuf) + // println("entry size", size) + entryData := make([]byte, size) + n, err = iter.r.Read(entryData) + if err != nil { + return + } + if n != int(size) { + return nil, fmt.Errorf("entry data %d bytes, expected %d bytes", n, size) + } + logEntry = &filer_pb.LogEntry{} + if err = proto.Unmarshal(entryData, logEntry); err != nil { + return + } + if logEntry.TsNs <= iter.startTsNs { + continue + } + if iter.stopTsNs != 0 && logEntry.TsNs > iter.stopTsNs { + return nil, io.EOF + } + return + } +} diff --git a/weed/filer/filerstore_wrapper.go b/weed/filer/filerstore_wrapper.go index d3beaba89..9c448edfd 100644 --- a/weed/filer/filerstore_wrapper.go +++ b/weed/filer/filerstore_wrapper.go @@ -85,7 +85,7 @@ func (fsw *FilerStoreWrapper) AddPathSpecificStore(path string, storeId string, func (fsw *FilerStoreWrapper) getActualStore(path util.FullPath) (store FilerStore) { store = fsw.defaultStore - if path == "/" { + if path == "/" || path == "//" { return } var storeId string @@ -182,7 +182,7 @@ func (fsw *FilerStoreWrapper) DeleteEntry(ctx context.Context, fp util.FullPath) }() existingEntry, findErr := fsw.FindEntry(ctx, fp) - if findErr == filer_pb.ErrNotFound { + if findErr == filer_pb.ErrNotFound || existingEntry == nil { return nil } if len(existingEntry.HardLinkId) != 0 { diff --git a/weed/filer/leveldb2/leveldb2_store.go b/weed/filer/leveldb2/leveldb2_store.go index 78d15382f..b465046f9 100644 --- a/weed/filer/leveldb2/leveldb2_store.go +++ b/weed/filer/leveldb2/leveldb2_store.go @@ -25,8 +25,9 @@ func init() { } type LevelDB2Store struct { - dbs []*leveldb.DB - dbCount int + dbs []*leveldb.DB + dbCount int + ReadOnly bool } func (store *LevelDB2Store) GetName() string { @@ -49,6 +50,7 @@ func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) { BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB Filter: filter.NewBloomFilter(8), // false positive rate 0.02 + ReadOnly: store.ReadOnly, } for d := 0; d < dbCount; d++ { diff --git a/weed/filer/leveldb3/leveldb3_store.go b/weed/filer/leveldb3/leveldb3_store.go index 406dc80be..2522221da 100644 --- a/weed/filer/leveldb3/leveldb3_store.go +++ b/weed/filer/leveldb3/leveldb3_store.go @@ -31,9 +31,10 @@ func init() { } type LevelDB3Store struct { - dir string - dbs map[string]*leveldb.DB - dbsLock sync.RWMutex + dir string + dbs map[string]*leveldb.DB + dbsLock sync.RWMutex + ReadOnly bool } func (store *LevelDB3Store) GetName() string { @@ -69,12 +70,14 @@ func (store *LevelDB3Store) loadDB(name string) (*leveldb.DB, error) { BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB Filter: bloom, + ReadOnly: store.ReadOnly, } if name != DEFAULT { opts = &opt.Options{ BlockCacheCapacity: 16 * 1024 * 1024, // default value is 8MiB WriteBuffer: 8 * 1024 * 1024, // default value is 4MiB Filter: bloom, + ReadOnly: store.ReadOnly, } } diff --git a/weed/filer/meta_replay.go b/weed/filer/meta_replay.go index 51c4e6987..0432e17de 100644 --- a/weed/filer/meta_replay.go +++ b/weed/filer/meta_replay.go @@ -2,6 +2,7 @@ package filer import ( "context" + "sync" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" @@ -35,3 +36,40 @@ func Replay(filerStore FilerStore, resp *filer_pb.SubscribeMetadataResponse) err return nil } + + +// ParallelProcessDirectoryStructure processes each entry in parallel, and also ensure parent directories are processed first. +// This also assumes the parent directories are in the entryChan already. +func ParallelProcessDirectoryStructure(entryChan chan *Entry, concurrency int, eachEntryFn func(entry *Entry)(error)) (firstErr error) { + + executors := util.NewLimitedConcurrentExecutor(concurrency) + + var wg sync.WaitGroup + for entry := range entryChan { + wg.Add(1) + if entry.IsDirectory() { + func() { + defer wg.Done() + if err := eachEntryFn(entry); err != nil { + if firstErr == nil { + firstErr = err + } + } + }() + } else { + executors.Execute(func() { + defer wg.Done() + if err := eachEntryFn(entry); err != nil { + if firstErr == nil { + firstErr = err + } + } + }) + } + if firstErr != nil { + break + } + } + wg.Wait() + return +} diff --git a/weed/filer/reader_cache.go b/weed/filer/reader_cache.go index 7be54b193..fea2bbc89 100644 --- a/weed/filer/reader_cache.go +++ b/weed/filer/reader_cache.go @@ -2,7 +2,6 @@ package filer import ( "fmt" - "github.com/seaweedfs/seaweedfs/weed/util" "sync" "sync/atomic" "time" @@ -10,6 +9,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/util/chunk_cache" "github.com/seaweedfs/seaweedfs/weed/util/mem" "github.com/seaweedfs/seaweedfs/weed/wdclient" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) type ReaderCache struct { @@ -171,7 +171,7 @@ func (s *SingleChunkCacher) startCaching() { s.data = mem.Allocate(s.chunkSize) - _, s.err = util.RetriedFetchChunkData(s.data, urlStrings, s.cipherKey, s.isGzipped, true, 0) + _, s.err = util_http.RetriedFetchChunkData(s.data, urlStrings, s.cipherKey, s.isGzipped, true, 0) if s.err != nil { mem.Free(s.data) s.data = nil diff --git a/weed/filer/redis/redis_cluster_store.go b/weed/filer/redis/redis_cluster_store.go index b24a9f5d3..be2710948 100644 --- a/weed/filer/redis/redis_cluster_store.go +++ b/weed/filer/redis/redis_cluster_store.go @@ -1,7 +1,7 @@ package redis import ( - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/util" ) diff --git a/weed/filer/redis/redis_store.go b/weed/filer/redis/redis_store.go index c3f7194e6..823bbf610 100644 --- a/weed/filer/redis/redis_store.go +++ b/weed/filer/redis/redis_store.go @@ -1,7 +1,7 @@ package redis import ( - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/util" ) diff --git a/weed/filer/redis/universal_redis_store.go b/weed/filer/redis/universal_redis_store.go index 33c0ea342..8c8d6539c 100644 --- a/weed/filer/redis/universal_redis_store.go +++ b/weed/filer/redis/universal_redis_store.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" diff --git a/weed/filer/redis/universal_redis_store_kv.go b/weed/filer/redis/universal_redis_store_kv.go index 12ab70cec..230cda759 100644 --- a/weed/filer/redis/universal_redis_store_kv.go +++ b/weed/filer/redis/universal_redis_store_kv.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/seaweedfs/seaweedfs/weed/filer" ) diff --git a/weed/filer/redis2/redis_cluster_store.go b/weed/filer/redis2/redis_cluster_store.go index 835bb5154..6e4f11d22 100644 --- a/weed/filer/redis2/redis_cluster_store.go +++ b/weed/filer/redis2/redis_cluster_store.go @@ -1,7 +1,7 @@ package redis2 import ( - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/util" ) diff --git a/weed/filer/redis2/redis_sentinel_store.go b/weed/filer/redis2/redis_sentinel_store.go index 313977dc7..5fc368fc7 100644 --- a/weed/filer/redis2/redis_sentinel_store.go +++ b/weed/filer/redis2/redis_sentinel_store.go @@ -1,7 +1,7 @@ package redis2 import ( - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/util" "time" diff --git a/weed/filer/redis2/redis_store.go b/weed/filer/redis2/redis_store.go index 52b29121d..a271dbcd1 100644 --- a/weed/filer/redis2/redis_store.go +++ b/weed/filer/redis2/redis_store.go @@ -1,7 +1,7 @@ package redis2 import ( - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/util" ) diff --git a/weed/filer/redis2/universal_redis_store.go b/weed/filer/redis2/universal_redis_store.go index 6b0e65c3d..d3f01f88a 100644 --- a/weed/filer/redis2/universal_redis_store.go +++ b/weed/filer/redis2/universal_redis_store.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" @@ -57,7 +57,7 @@ func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer } if name != "" { - if err = store.Client.ZAddNX(ctx, genDirectoryListKey(dir), &redis.Z{Score: 0, Member: name}).Err(); err != nil { + if err = store.Client.ZAddNX(ctx, genDirectoryListKey(dir), redis.Z{Score: 0, Member: name}).Err(); err != nil { return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err) } } diff --git a/weed/filer/redis2/universal_redis_store_kv.go b/weed/filer/redis2/universal_redis_store_kv.go index a68a3449c..5b515f605 100644 --- a/weed/filer/redis2/universal_redis_store_kv.go +++ b/weed/filer/redis2/universal_redis_store_kv.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/seaweedfs/seaweedfs/weed/filer" ) diff --git a/weed/filer/redis3/ItemList.go b/weed/filer/redis3/ItemList.go index c576cbeb4..9e38089a7 100644 --- a/weed/filer/redis3/ItemList.go +++ b/weed/filer/redis3/ItemList.go @@ -4,7 +4,7 @@ import ( "bytes" "context" "fmt" - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/seaweedfs/seaweedfs/weed/util/skiplist" ) @@ -399,9 +399,9 @@ func (nl *ItemList) NodeSize(node *skiplist.SkipListElementReference) int { func (nl *ItemList) NodeAddMember(node *skiplist.SkipListElementReference, names ...string) error { key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer) - var members []*redis.Z + var members []redis.Z for _, name := range names { - members = append(members, &redis.Z{ + members = append(members, redis.Z{ Score: 0, Member: name, }) diff --git a/weed/filer/redis3/item_list_serde.go b/weed/filer/redis3/item_list_serde.go index e71243838..f4410b61b 100644 --- a/weed/filer/redis3/item_list_serde.go +++ b/weed/filer/redis3/item_list_serde.go @@ -1,7 +1,7 @@ package redis3 import ( - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/util/skiplist" "google.golang.org/protobuf/proto" diff --git a/weed/filer/redis3/kv_directory_children.go b/weed/filer/redis3/kv_directory_children.go index 422500ed7..5a2d76141 100644 --- a/weed/filer/redis3/kv_directory_children.go +++ b/weed/filer/redis3/kv_directory_children.go @@ -3,7 +3,7 @@ package redis3 import ( "context" "fmt" - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/seaweedfs/seaweedfs/weed/glog" ) diff --git a/weed/filer/redis3/kv_directory_children_test.go b/weed/filer/redis3/kv_directory_children_test.go index 9d7acacf1..76a8dc00f 100644 --- a/weed/filer/redis3/kv_directory_children_test.go +++ b/weed/filer/redis3/kv_directory_children_test.go @@ -3,7 +3,7 @@ package redis3 import ( "context" "fmt" - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/stvp/tempredis" "strconv" "testing" @@ -116,7 +116,7 @@ func BenchmarkRedis(b *testing.B) { }) for i := 0; i < b.N; i++ { - client.ZAddNX(context.Background(), "/yyy/bin", &redis.Z{Score: 0, Member: strconv.Itoa(i) + "namexxxxxxxxxxxxxxxxxxx"}) + client.ZAddNX(context.Background(), "/yyy/bin", redis.Z{Score: 0, Member: strconv.Itoa(i) + "namexxxxxxxxxxxxxxxxxxx"}) } } @@ -149,7 +149,7 @@ func xTestNameListAdd(t *testing.T) { ts1 := time.Now() for i := 0; i < N; i++ { - client.ZAddNX(context.Background(), "/x", &redis.Z{Score: 0, Member: fmt.Sprintf("name %8d", i)}) + client.ZAddNX(context.Background(), "/x", redis.Z{Score: 0, Member: fmt.Sprintf("name %8d", i)}) } ts2 := time.Now() @@ -205,6 +205,6 @@ func xBenchmarkRedis(b *testing.B) { }) for i := 0; i < b.N; i++ { - client.ZAddNX(context.Background(), "/xxx/bin", &redis.Z{Score: 0, Member: fmt.Sprintf("name %8d", i)}) + client.ZAddNX(context.Background(), "/xxx/bin", redis.Z{Score: 0, Member: fmt.Sprintf("name %8d", i)}) } } diff --git a/weed/filer/redis3/redis_cluster_store.go b/weed/filer/redis3/redis_cluster_store.go index 1efa8e9b0..a8858bfa9 100644 --- a/weed/filer/redis3/redis_cluster_store.go +++ b/weed/filer/redis3/redis_cluster_store.go @@ -1,9 +1,9 @@ package redis3 import ( - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/go-redsync/redsync/v4" - "github.com/go-redsync/redsync/v4/redis/goredis/v8" + "github.com/go-redsync/redsync/v4/redis/goredis/v9" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/util" ) diff --git a/weed/filer/redis3/redis_sentinel_store.go b/weed/filer/redis3/redis_sentinel_store.go index 4135d9956..2e0a8d4ae 100644 --- a/weed/filer/redis3/redis_sentinel_store.go +++ b/weed/filer/redis3/redis_sentinel_store.go @@ -3,9 +3,9 @@ package redis3 import ( "time" - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/go-redsync/redsync/v4" - "github.com/go-redsync/redsync/v4/redis/goredis/v8" + "github.com/go-redsync/redsync/v4/redis/goredis/v9" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/util" ) diff --git a/weed/filer/redis3/redis_store.go b/weed/filer/redis3/redis_store.go index 9b98459e6..339115563 100644 --- a/weed/filer/redis3/redis_store.go +++ b/weed/filer/redis3/redis_store.go @@ -1,9 +1,9 @@ package redis3 import ( - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/go-redsync/redsync/v4" - "github.com/go-redsync/redsync/v4/redis/goredis/v8" + "github.com/go-redsync/redsync/v4/redis/goredis/v9" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/util" ) diff --git a/weed/filer/redis3/skiplist_element_store.go b/weed/filer/redis3/skiplist_element_store.go index b589e3e77..46506187e 100644 --- a/weed/filer/redis3/skiplist_element_store.go +++ b/weed/filer/redis3/skiplist_element_store.go @@ -3,7 +3,7 @@ package redis3 import ( "context" "fmt" - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/util/skiplist" "google.golang.org/protobuf/proto" diff --git a/weed/filer/redis3/universal_redis_store.go b/weed/filer/redis3/universal_redis_store.go index 2fb9a5b3f..51675d971 100644 --- a/weed/filer/redis3/universal_redis_store.go +++ b/weed/filer/redis3/universal_redis_store.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" redsync "github.com/go-redsync/redsync/v4" "github.com/seaweedfs/seaweedfs/weed/filer" diff --git a/weed/filer/redis3/universal_redis_store_kv.go b/weed/filer/redis3/universal_redis_store_kv.go index 280eb4a96..fd665d5d4 100644 --- a/weed/filer/redis3/universal_redis_store_kv.go +++ b/weed/filer/redis3/universal_redis_store_kv.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/seaweedfs/seaweedfs/weed/filer" ) diff --git a/weed/filer/redis_lua/redis_cluster_store.go b/weed/filer/redis_lua/redis_cluster_store.go index ae274bb09..251aadbcd 100644 --- a/weed/filer/redis_lua/redis_cluster_store.go +++ b/weed/filer/redis_lua/redis_cluster_store.go @@ -1,7 +1,7 @@ package redis_lua import ( - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/util" ) diff --git a/weed/filer/redis_lua/redis_sentinel_store.go b/weed/filer/redis_lua/redis_sentinel_store.go index ac8ae75d7..f22a7fa66 100644 --- a/weed/filer/redis_lua/redis_sentinel_store.go +++ b/weed/filer/redis_lua/redis_sentinel_store.go @@ -1,7 +1,7 @@ package redis_lua import ( - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/util" "time" diff --git a/weed/filer/redis_lua/redis_store.go b/weed/filer/redis_lua/redis_store.go index e88f3a3dc..8574baa09 100644 --- a/weed/filer/redis_lua/redis_store.go +++ b/weed/filer/redis_lua/redis_store.go @@ -1,7 +1,7 @@ package redis_lua import ( - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/util" ) diff --git a/weed/filer/redis_lua/stored_procedure/init.go b/weed/filer/redis_lua/stored_procedure/init.go index 1412ceba2..9373cc5a3 100644 --- a/weed/filer/redis_lua/stored_procedure/init.go +++ b/weed/filer/redis_lua/stored_procedure/init.go @@ -2,7 +2,7 @@ package stored_procedure import ( _ "embed" - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" ) func init() { diff --git a/weed/filer/redis_lua/universal_redis_store.go b/weed/filer/redis_lua/universal_redis_store.go index 59c128030..9e8dbcda7 100644 --- a/weed/filer/redis_lua/universal_redis_store.go +++ b/weed/filer/redis_lua/universal_redis_store.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/filer/redis_lua/stored_procedure" diff --git a/weed/filer/redis_lua/universal_redis_store_kv.go b/weed/filer/redis_lua/universal_redis_store_kv.go index a03ae6573..01a7ba560 100644 --- a/weed/filer/redis_lua/universal_redis_store_kv.go +++ b/weed/filer/redis_lua/universal_redis_store_kv.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/go-redis/redis/v8" + "github.com/redis/go-redis/v9" "github.com/seaweedfs/seaweedfs/weed/filer" ) diff --git a/weed/filer/stream.go b/weed/filer/stream.go index 23a853b9a..fdb443b53 100644 --- a/weed/filer/stream.go +++ b/weed/filer/stream.go @@ -16,6 +16,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/wdclient" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var getLookupFileIdBackoffSchedule = []time.Duration{ @@ -194,7 +195,7 @@ func ReadAll(buffer []byte, masterClient *wdclient.MasterClient, chunks []*filer return err } - n, err := util.RetriedFetchChunkData(buffer[idx:idx+int(chunkView.ViewSize)], urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.OffsetInChunk) + n, err := util_http.RetriedFetchChunkData(buffer[idx:idx+int(chunkView.ViewSize)], urlStrings, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.OffsetInChunk) if err != nil { return err } @@ -350,7 +351,7 @@ func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error { var buffer bytes.Buffer var shouldRetry bool for _, urlString := range urlStrings { - shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.OffsetInChunk, int(chunkView.ViewSize), func(data []byte) { + shouldRetry, err = util_http.ReadUrlAsStream(urlString+"?readDeleted=true", chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.OffsetInChunk, int(chunkView.ViewSize), func(data []byte) { buffer.Write(data) }) if !shouldRetry { diff --git a/weed/iamapi/iamapi_server.go b/weed/iamapi/iamapi_server.go index 3a4fb914e..d1575a14e 100644 --- a/weed/iamapi/iamapi_server.go +++ b/weed/iamapi/iamapi_server.go @@ -72,7 +72,7 @@ func (iama *IamApiServer) registerRouter(router *mux.Router) { // ListBuckets // apiRouter.Methods("GET").Path("/").HandlerFunc(track(s3a.iam.Auth(s3a.ListBucketsHandler, ACTION_ADMIN), "LIST")) - apiRouter.Methods("POST").Path("/").HandlerFunc(iama.iam.Auth(iama.DoActions, ACTION_ADMIN)) + apiRouter.Methods(http.MethodPost).Path("/").HandlerFunc(iama.iam.Auth(iama.DoActions, ACTION_ADMIN)) // // NotFound apiRouter.NotFoundHandler = http.HandlerFunc(s3err.NotFoundHandler) diff --git a/weed/iamapi/iamapi_test.go b/weed/iamapi/iamapi_test.go index efd74a059..f32e1ac51 100644 --- a/weed/iamapi/iamapi_test.go +++ b/weed/iamapi/iamapi_test.go @@ -189,7 +189,7 @@ func TestDeleteUser(t *testing.T) { func executeRequest(req *http.Request, v interface{}) (*httptest.ResponseRecorder, error) { rr := httptest.NewRecorder() apiRouter := mux.NewRouter().SkipClean(true) - apiRouter.Path("/").Methods("POST").HandlerFunc(ias.DoActions) + apiRouter.Path("/").Methods(http.MethodPost).HandlerFunc(ias.DoActions) apiRouter.ServeHTTP(rr, req) return rr, xml.Unmarshal(rr.Body.Bytes(), &v) } diff --git a/weed/mount/filehandle.go b/weed/mount/filehandle.go index 2e08432c0..f47d4a877 100644 --- a/weed/mount/filehandle.go +++ b/weed/mount/filehandle.go @@ -66,8 +66,8 @@ func (fh *FileHandle) FullPath() util.FullPath { return fp } -func (fh *FileHandle) GetEntry() *filer_pb.Entry { - return fh.entry.GetEntry() +func (fh *FileHandle) GetEntry() *LockedEntry { + return fh.entry } func (fh *FileHandle) SetEntry(entry *filer_pb.Entry) { @@ -90,13 +90,6 @@ func (fh *FileHandle) UpdateEntry(fn func(entry *filer_pb.Entry)) *filer_pb.Entr } func (fh *FileHandle) AddChunks(chunks []*filer_pb.FileChunk) { - fh.entryLock.Lock() - defer fh.entryLock.Unlock() - - if fh.entry == nil { - return - } - fh.entry.AppendChunks(chunks) } @@ -105,9 +98,6 @@ func (fh *FileHandle) ReleaseHandle() { fhActiveLock := fh.wfs.fhLockTable.AcquireLock("ReleaseHandle", fh.fh, util.ExclusiveLock) defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock) - fh.entryLock.Lock() - defer fh.entryLock.Unlock() - fh.dirtyPages.Destroy() if IsDebugFileReadWrite { fh.mirrorFile.Close() diff --git a/weed/mount/filehandle_map.go b/weed/mount/filehandle_map.go index f0051f061..852ef9e35 100644 --- a/weed/mount/filehandle_map.go +++ b/weed/mount/filehandle_map.go @@ -1,6 +1,7 @@ package mount import ( + "github.com/seaweedfs/seaweedfs/weed/util" "sync" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" @@ -8,7 +9,6 @@ import ( type FileHandleToInode struct { sync.RWMutex - nextFh FileHandleId inode2fh map[uint64]*FileHandle fh2inode map[FileHandleId]uint64 } @@ -17,7 +17,6 @@ func NewFileHandleToInode() *FileHandleToInode { return &FileHandleToInode{ inode2fh: make(map[uint64]*FileHandle), fh2inode: make(map[FileHandleId]uint64), - nextFh: 0, } } @@ -43,14 +42,13 @@ func (i *FileHandleToInode) AcquireFileHandle(wfs *WFS, inode uint64, entry *fil defer i.Unlock() fh, found := i.inode2fh[inode] if !found { - fh = newFileHandle(wfs, i.nextFh, inode, entry) - i.nextFh++ + fh = newFileHandle(wfs, FileHandleId(util.RandomUint64()), inode, entry) i.inode2fh[inode] = fh i.fh2inode[fh.fh] = inode } else { fh.counter++ } - if fh.GetEntry() != entry { + if fh.GetEntry().GetEntry() != entry { fh.SetEntry(entry) } return fh diff --git a/weed/mount/filehandle_read.go b/weed/mount/filehandle_read.go index 7b2629c13..3c315b1c4 100644 --- a/weed/mount/filehandle_read.go +++ b/weed/mount/filehandle_read.go @@ -29,23 +29,19 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, int64, e fileFullPath := fh.FullPath() entry := fh.GetEntry() - if entry == nil { - return 0, 0, io.EOF - } if entry.IsInRemoteOnly() { glog.V(4).Infof("download remote entry %s", fileFullPath) - newEntry, err := fh.downloadRemoteEntry(entry) + err := fh.downloadRemoteEntry(entry) if err != nil { glog.V(1).Infof("download remote entry %s: %v", fileFullPath, err) return 0, 0, err } - entry = newEntry } fileSize := int64(entry.Attributes.FileSize) if fileSize == 0 { - fileSize = int64(filer.FileSize(entry)) + fileSize = int64(filer.FileSize(entry.GetEntry())) } if fileSize == 0 { @@ -70,7 +66,7 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, int64, e return int64(totalRead), ts, err } -func (fh *FileHandle) downloadRemoteEntry(entry *filer_pb.Entry) (*filer_pb.Entry, error) { +func (fh *FileHandle) downloadRemoteEntry(entry *LockedEntry) (error) { fileFullPath := fh.FullPath() dir, _ := fileFullPath.DirAndName() @@ -88,12 +84,12 @@ func (fh *FileHandle) downloadRemoteEntry(entry *filer_pb.Entry) (*filer_pb.Entr return fmt.Errorf("CacheRemoteObjectToLocalCluster file %s: %v", fileFullPath, err) } - entry = resp.Entry + entry.SetEntry(resp.Entry) fh.wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, resp.Entry)) return nil }) - return entry, err + return err } diff --git a/weed/mount/locked_entry.go b/weed/mount/locked_entry.go index f3b4bf484..c5fbaee91 100644 --- a/weed/mount/locked_entry.go +++ b/weed/mount/locked_entry.go @@ -16,6 +16,8 @@ func (le *LockedEntry) GetEntry() *filer_pb.Entry { return le.Entry } +// SetEntry sets the entry of the LockedEntry +// entry should never be nil func (le *LockedEntry) SetEntry(entry *filer_pb.Entry) { le.Lock() defer le.Unlock() diff --git a/weed/mount/meta_cache/meta_cache.go b/weed/mount/meta_cache/meta_cache.go index 46a76365c..8a571d9e6 100644 --- a/weed/mount/meta_cache/meta_cache.go +++ b/weed/mount/meta_cache/meta_cache.go @@ -4,6 +4,7 @@ import ( "context" "os" "sync" + "time" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/filer/leveldb" @@ -118,6 +119,9 @@ func (mc *MetaCache) FindEntry(ctx context.Context, fp util.FullPath) (entry *fi if err != nil { return nil, err } + if entry.TtlSec > 0 && entry.Crtime.Add(time.Duration(entry.TtlSec)).Before(time.Now()) { + return nil, filer_pb.ErrNotFound + } mc.mapIdFromFilerToLocal(entry) return } @@ -143,6 +147,9 @@ func (mc *MetaCache) ListDirectoryEntries(ctx context.Context, dirPath util.Full } _, err := mc.localStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, func(entry *filer.Entry) bool { + if entry.TtlSec > 0 && entry.Crtime.Add(time.Duration(entry.TtlSec)).Before(time.Now()) { + return true + } mc.mapIdFromFilerToLocal(entry) return eachEntryFunc(entry) }) diff --git a/weed/mount/meta_cache/meta_cache_subscribe.go b/weed/mount/meta_cache/meta_cache_subscribe.go index 72da0ca64..d3bb27d08 100644 --- a/weed/mount/meta_cache/meta_cache_subscribe.go +++ b/weed/mount/meta_cache/meta_cache_subscribe.go @@ -7,6 +7,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/util" + "strings" ) func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.FilerClient, dir string, lastTsNs int64) error { @@ -57,12 +58,17 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil } + prefix := dir + if !strings.HasSuffix(prefix, "/") { + prefix = prefix + "/" + } + metadataFollowOption := &pb.MetadataFollowOption{ ClientName: "mount", ClientId: selfSignature, ClientEpoch: 1, SelfSignature: selfSignature, - PathPrefix: dir, + PathPrefix: prefix, AdditionalPathPrefixes: nil, DirectoriesToWatch: nil, StartTsNs: lastTsNs, diff --git a/weed/mount/weedfs.go b/weed/mount/weedfs.go index c5a1d2755..a9fbd9380 100644 --- a/weed/mount/weedfs.go +++ b/weed/mount/weedfs.go @@ -112,9 +112,6 @@ func NewSeaweedFileSystem(option *Option) *WFS { fhActiveLock := fh.wfs.fhLockTable.AcquireLock("invalidateFunc", fh.fh, util.ExclusiveLock) defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock) - fh.entryLock.Lock() - defer fh.entryLock.Unlock() - // Recreate dirty pages fh.dirtyPages.Destroy() fh.dirtyPages = newPageWriter(fh, wfs.option.ChunkSizeLimit) @@ -122,7 +119,7 @@ func NewSeaweedFileSystem(option *Option) *WFS { // Update handle entry newentry, status := wfs.maybeLoadEntry(filePath) if status == fuse.OK { - if fh.GetEntry() != newentry { + if fh.GetEntry().GetEntry() != newentry { fh.SetEntry(newentry) } } diff --git a/weed/mount/weedfs_dir_lookup.go b/weed/mount/weedfs_dir_lookup.go index e646b06a9..f3ba0cc85 100644 --- a/weed/mount/weedfs_dir_lookup.go +++ b/weed/mount/weedfs_dir_lookup.go @@ -59,7 +59,7 @@ func (wfs *WFS) Lookup(cancel <-chan struct{}, header *fuse.InHeader, name strin if fh, found := wfs.fhmap.FindFileHandle(inode); found { fh.entryLock.RLock() - if entry := fh.GetEntry(); entry != nil { + if entry := fh.GetEntry().GetEntry(); entry != nil { glog.V(4).Infof("lookup opened file %s size %d", dirPath.Child(localEntry.Name()), filer.FileSize(entry)) localEntry = filer.FromPbEntry(string(dirPath), entry) } diff --git a/weed/mount/weedfs_dir_read.go b/weed/mount/weedfs_dir_read.go index f140fd86f..c80ecce9e 100644 --- a/weed/mount/weedfs_dir_read.go +++ b/weed/mount/weedfs_dir_read.go @@ -6,6 +6,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/mount/meta_cache" + "github.com/seaweedfs/seaweedfs/weed/util" "math" "sync" ) @@ -43,10 +44,7 @@ func NewDirectoryHandleToInode() *DirectoryHandleToInode { } func (wfs *WFS) AcquireDirectoryHandle() (DirectoryHandleId, *DirectoryHandle) { - wfs.fhmap.Lock() - fh := wfs.fhmap.nextFh - wfs.fhmap.nextFh++ - wfs.fhmap.Unlock() + fh := FileHandleId(util.RandomUint64()) wfs.dhmap.Lock() defer wfs.dhmap.Unlock() @@ -173,7 +171,7 @@ func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out *fuse.DirEntryList, isPl } if fh, found := wfs.fhmap.FindFileHandle(inode); found { glog.V(4).Infof("readdir opened file %s", dirPath.Child(dirEntry.Name)) - entry = filer.FromPbEntry(string(dirPath), fh.GetEntry()) + entry = filer.FromPbEntry(string(dirPath), fh.GetEntry().GetEntry()) } wfs.outputFilerEntry(entryOut, inode, entry) } diff --git a/weed/mount/weedfs_file_lseek.go b/weed/mount/weedfs_file_lseek.go index 35157d993..0cf7ef43b 100644 --- a/weed/mount/weedfs_file_lseek.go +++ b/weed/mount/weedfs_file_lseek.go @@ -38,10 +38,8 @@ func (wfs *WFS) Lseek(cancel <-chan struct{}, in *fuse.LseekIn, out *fuse.LseekO // lock the file until the proper offset was calculated fhActiveLock := fh.wfs.fhLockTable.AcquireLock("Lseek", fh.fh, util.SharedLock) defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock) - fh.entryLock.RLock() - defer fh.entryLock.RUnlock() - fileSize := int64(filer.FileSize(fh.GetEntry())) + fileSize := int64(filer.FileSize(fh.GetEntry().GetEntry())) offset := max(int64(in.Offset), 0) glog.V(4).Infof( diff --git a/weed/mount/weedfs_file_sync.go b/weed/mount/weedfs_file_sync.go index 762a9b8de..d857606bd 100644 --- a/weed/mount/weedfs_file_sync.go +++ b/weed/mount/weedfs_file_sync.go @@ -116,13 +116,8 @@ func (wfs *WFS) doFlush(fh *FileHandle, uid, gid uint32) fuse.Status { defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock) err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { - fh.entryLock.Lock() - defer fh.entryLock.Unlock() entry := fh.GetEntry() - if entry == nil { - return nil - } entry.Name = name // this flush may be just after a rename operation if entry.Attributes != nil { @@ -141,7 +136,7 @@ func (wfs *WFS) doFlush(fh *FileHandle, uid, gid uint32) fuse.Status { request := &filer_pb.CreateEntryRequest{ Directory: string(dir), - Entry: entry, + Entry: entry.GetEntry(), Signatures: []int32{wfs.signature}, SkipCheckParentDirectory: true, } diff --git a/weed/mount/weedfs_write.go b/weed/mount/weedfs_write.go index 4c8470245..77ad01b89 100644 --- a/weed/mount/weedfs_write.go +++ b/weed/mount/weedfs_write.go @@ -14,8 +14,12 @@ import ( func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath) filer.SaveDataAsChunkFunctionType { return func(reader io.Reader, filename string, offset int64, tsNs int64) (chunk *filer_pb.FileChunk, err error) { + uploader, err := operation.NewUploader() + if err != nil { + return + } - fileId, uploadResult, err, data := operation.UploadWithRetry( + fileId, uploadResult, err, data := uploader.UploadWithRetry( wfs, &filer_pb.AssignVolumeRequest{ Count: 1, diff --git a/weed/mq/broker/broker_topic_partition_read_write.go b/weed/mq/broker/broker_topic_partition_read_write.go index 7dd78c582..4c1b9a1e2 100644 --- a/weed/mq/broker/broker_topic_partition_read_write.go +++ b/weed/mq/broker/broker_topic_partition_read_write.go @@ -13,6 +13,7 @@ import ( "math" "sync/atomic" "time" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) func (b *MessageQueueBroker) genLogFlushFunc(t topic.Topic, partition *mq_pb.Partition) log_buffer.LogFlushFuncType { @@ -130,7 +131,7 @@ func (b *MessageQueueBroker) genLogOnDiskReadFunc(t topic.Topic, partition *mq_p for _, urlString := range urlStrings { // TODO optimization opportunity: reuse the buffer var data []byte - if data, _, err = util.Get(urlString); err == nil { + if data, _, err = util_http.Get(urlString); err == nil { processed = true if processedTsNs, err = eachChunkFn(data, eachLogEntryFn, starTsNs, stopTsNs); err != nil { return diff --git a/weed/mq/broker/broker_write.go b/weed/mq/broker/broker_write.go index 896f0ee75..9999529fb 100644 --- a/weed/mq/broker/broker_write.go +++ b/weed/mq/broker/broker_write.go @@ -55,7 +55,13 @@ func (b *MessageQueueBroker) appendToFile(targetFile string, data []byte) error func (b *MessageQueueBroker) assignAndUpload(targetFile string, data []byte) (fileId string, uploadResult *operation.UploadResult, err error) { reader := util.NewBytesReader(data) - fileId, uploadResult, err, _ = operation.UploadWithRetry( + + uploader, err := operation.NewUploader() + if err != nil { + return + } + + fileId, uploadResult, err, _ = uploader.UploadWithRetry( b, &filer_pb.AssignVolumeRequest{ Count: 1, diff --git a/weed/mq/client/cmd/weed_pub_kv/publisher_kv.go b/weed/mq/client/cmd/weed_pub_kv/publisher_kv.go index 096b355a1..3ab3cb251 100644 --- a/weed/mq/client/cmd/weed_pub_kv/publisher_kv.go +++ b/weed/mq/client/cmd/weed_pub_kv/publisher_kv.go @@ -9,6 +9,7 @@ import ( "strings" "sync" "time" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var ( @@ -45,6 +46,7 @@ func doPublish(publisher *pub_client.TopicPublisher, id int) { func main() { flag.Parse() + util_http.InitGlobalHttpClient() config := &pub_client.PublisherConfiguration{ Topic: topic.NewTopic(*namespace, *t), diff --git a/weed/mq/client/cmd/weed_pub_record/publisher_record.go b/weed/mq/client/cmd/weed_pub_record/publisher_record.go index a5fbd455e..f340dd1c8 100644 --- a/weed/mq/client/cmd/weed_pub_record/publisher_record.go +++ b/weed/mq/client/cmd/weed_pub_record/publisher_record.go @@ -11,6 +11,7 @@ import ( "strings" "sync" "time" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var ( @@ -86,6 +87,7 @@ func (r *MyRecord) ToRecordValue() *schema_pb.RecordValue { func main() { flag.Parse() + util_http.InitGlobalHttpClient() recordType := schema.RecordTypeBegin(). WithField("key", schema.TypeBytes). diff --git a/weed/mq/client/cmd/weed_sub_kv/subscriber_kv.go b/weed/mq/client/cmd/weed_sub_kv/subscriber_kv.go index 902e7ed1b..8ff667763 100644 --- a/weed/mq/client/cmd/weed_sub_kv/subscriber_kv.go +++ b/weed/mq/client/cmd/weed_sub_kv/subscriber_kv.go @@ -11,6 +11,7 @@ import ( "google.golang.org/grpc/credentials/insecure" "strings" "time" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var ( @@ -25,6 +26,7 @@ var ( func main() { flag.Parse() + util_http.InitGlobalHttpClient() subscriberConfig := &sub_client.SubscriberConfiguration{ ConsumerGroup: "test", diff --git a/weed/mq/client/cmd/weed_sub_record/subscriber_record.go b/weed/mq/client/cmd/weed_sub_record/subscriber_record.go index 674c881ba..00fe83feb 100644 --- a/weed/mq/client/cmd/weed_sub_record/subscriber_record.go +++ b/weed/mq/client/cmd/weed_sub_record/subscriber_record.go @@ -13,6 +13,7 @@ import ( "google.golang.org/protobuf/proto" "strings" "time" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var ( @@ -51,6 +52,7 @@ func FromSchemaRecordValue(recordValue *schema_pb.RecordValue) *MyRecord { func main() { flag.Parse() + util_http.InitGlobalHttpClient() subscriberConfig := &sub_client.SubscriberConfiguration{ ConsumerGroup: "test", diff --git a/weed/operation/chunked_file.go b/weed/operation/chunked_file.go index c451420fe..be3e5c98e 100644 --- a/weed/operation/chunked_file.go +++ b/weed/operation/chunked_file.go @@ -15,6 +15,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/util" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var ( @@ -79,7 +80,7 @@ func (cm *ChunkManifest) DeleteChunks(masterFn GetMasterFn, usePublicUrl bool, g for _, ci := range cm.Chunks { fileIds = append(fileIds, ci.Fid) } - results, err := DeleteFiles(masterFn, usePublicUrl, grpcDialOption, fileIds) + results, err := DeleteFileIds(masterFn, usePublicUrl, grpcDialOption, fileIds) if err != nil { glog.V(0).Infof("delete %+v: %v", fileIds, err) return fmt.Errorf("chunk delete: %v", err) @@ -95,7 +96,7 @@ func (cm *ChunkManifest) DeleteChunks(masterFn GetMasterFn, usePublicUrl bool, g } func readChunkNeedle(fileUrl string, w io.Writer, offset int64, jwt string) (written int64, e error) { - req, err := http.NewRequest("GET", fileUrl, nil) + req, err := http.NewRequest(http.MethodGet, fileUrl, nil) if err != nil { return written, err } @@ -103,11 +104,11 @@ func readChunkNeedle(fileUrl string, w io.Writer, offset int64, jwt string) (wri req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset)) } - resp, err := util.Do(req) + resp, err := util_http.Do(req) if err != nil { return written, err } - defer util.CloseResponse(resp) + defer util_http.CloseResponse(resp) switch resp.StatusCode { case http.StatusRequestedRangeNotSatisfiable: diff --git a/weed/operation/delete_content.go b/weed/operation/delete_content.go index cee80fb47..419223165 100644 --- a/weed/operation/delete_content.go +++ b/weed/operation/delete_content.go @@ -28,8 +28,8 @@ func ParseFileId(fid string) (vid string, key_cookie string, err error) { return fid[:commaIndex], fid[commaIndex+1:], nil } -// DeleteFiles batch deletes a list of fileIds -func DeleteFiles(masterFn GetMasterFn, usePublicUrl bool, grpcDialOption grpc.DialOption, fileIds []string) ([]*volume_server_pb.DeleteResult, error) { +// DeleteFileIds batch deletes a list of fileIds +func DeleteFileIds(masterFn GetMasterFn, usePublicUrl bool, grpcDialOption grpc.DialOption, fileIds []string) ([]*volume_server_pb.DeleteResult, error) { lookupFunc := func(vids []string) (results map[string]*LookupResult, err error) { results, err = LookupVolumeIds(masterFn, grpcDialOption, vids) @@ -43,11 +43,11 @@ func DeleteFiles(masterFn GetMasterFn, usePublicUrl bool, grpcDialOption grpc.Di return } - return DeleteFilesWithLookupVolumeId(grpcDialOption, fileIds, lookupFunc) + return DeleteFileIdsWithLookupVolumeId(grpcDialOption, fileIds, lookupFunc) } -func DeleteFilesWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []string, lookupFunc func(vid []string) (map[string]*LookupResult, error)) ([]*volume_server_pb.DeleteResult, error) { +func DeleteFileIdsWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []string, lookupFunc func(vid []string) (map[string]*LookupResult, error)) ([]*volume_server_pb.DeleteResult, error) { var ret []*volume_server_pb.DeleteResult @@ -102,7 +102,7 @@ func DeleteFilesWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []str go func(server pb.ServerAddress, fidList []string) { defer wg.Done() - if deleteResults, deleteErr := DeleteFilesAtOneVolumeServer(server, grpcDialOption, fidList, false); deleteErr != nil { + if deleteResults, deleteErr := DeleteFileIdsAtOneVolumeServer(server, grpcDialOption, fidList, false); deleteErr != nil { err = deleteErr } else if deleteResults != nil { resultChan <- deleteResults @@ -120,8 +120,8 @@ func DeleteFilesWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []str return ret, err } -// DeleteFilesAtOneVolumeServer deletes a list of files that is on one volume server via gRpc -func DeleteFilesAtOneVolumeServer(volumeServer pb.ServerAddress, grpcDialOption grpc.DialOption, fileIds []string, includeCookie bool) (ret []*volume_server_pb.DeleteResult, err error) { +// DeleteFileIdsAtOneVolumeServer deletes a list of files that is on one volume server via gRpc +func DeleteFileIdsAtOneVolumeServer(volumeServer pb.ServerAddress, grpcDialOption grpc.DialOption, fileIds []string, includeCookie bool) (ret []*volume_server_pb.DeleteResult, err error) { err = WithVolumeServerClient(false, volumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { diff --git a/weed/operation/needle_parse_test.go b/weed/operation/needle_parse_test.go index 07b0153a9..b4bac5976 100644 --- a/weed/operation/needle_parse_test.go +++ b/weed/operation/needle_parse_test.go @@ -38,15 +38,11 @@ If the content is already compressed, need to know the content size. */ func TestCreateNeedleFromRequest(t *testing.T) { - mc := &MockClient{} - tmp := HttpClient - HttpClient = mc - defer func() { - HttpClient = tmp - }() + mockClient := &MockClient{} + uploader := newUploader(mockClient) { - mc.needleHandling = func(n *needle.Needle, originalSize int, err error) { + mockClient.needleHandling = func(n *needle.Needle, originalSize int, err error) { assert.Equal(t, nil, err, "upload: %v", err) assert.Equal(t, "", string(n.Mime), "mime detection failed: %v", string(n.Mime)) assert.Equal(t, true, n.IsCompressed(), "this should be compressed") @@ -62,7 +58,7 @@ func TestCreateNeedleFromRequest(t *testing.T) { PairMap: nil, Jwt: "", } - uploadResult, err, data := Upload(bytes.NewReader([]byte(textContent)), uploadOption) + uploadResult, err, data := uploader.Upload(bytes.NewReader([]byte(textContent)), uploadOption) if len(data) != len(textContent) { t.Errorf("data actual %d expected %d", len(data), len(textContent)) } @@ -73,7 +69,7 @@ func TestCreateNeedleFromRequest(t *testing.T) { } { - mc.needleHandling = func(n *needle.Needle, originalSize int, err error) { + mockClient.needleHandling = func(n *needle.Needle, originalSize int, err error) { assert.Equal(t, nil, err, "upload: %v", err) assert.Equal(t, "text/plain", string(n.Mime), "mime detection failed: %v", string(n.Mime)) assert.Equal(t, true, n.IsCompressed(), "this should be compressed") @@ -90,7 +86,7 @@ func TestCreateNeedleFromRequest(t *testing.T) { PairMap: nil, Jwt: "", } - Upload(bytes.NewReader(gzippedData), uploadOption) + uploader.Upload(bytes.NewReader(gzippedData), uploadOption) } /* diff --git a/weed/operation/submit.go b/weed/operation/submit.go index 57bd81b14..516478dbe 100644 --- a/weed/operation/submit.go +++ b/weed/operation/submit.go @@ -217,7 +217,13 @@ func (fi FilePart) Upload(maxMB int, masterFn GetMasterFn, usePublicUrl bool, jw PairMap: nil, Jwt: jwt, } - ret, e, _ := Upload(fi.Reader, uploadOption) + + uploader, e := NewUploader() + if e != nil { + return 0, e + } + + ret, e, _ := uploader.Upload(fi.Reader, uploadOption) if e != nil { return 0, e } @@ -239,7 +245,13 @@ func upload_one_chunk(filename string, reader io.Reader, masterFn GetMasterFn, PairMap: nil, Jwt: jwt, } - uploadResult, uploadError, _ := Upload(reader, uploadOption) + + uploader, uploaderError := NewUploader() + if uploaderError != nil { + return 0, uploaderError + } + + uploadResult, uploadError, _ := uploader.Upload(reader, uploadOption) if uploadError != nil { return 0, uploadError } @@ -265,6 +277,12 @@ func upload_chunked_file_manifest(fileUrl string, manifest *ChunkManifest, jwt s PairMap: nil, Jwt: jwt, } - _, e = UploadData(buf, uploadOption) + + uploader, e := NewUploader() + if e != nil { + return e + } + + _, e = uploader.UploadData(buf, uploadOption) return e } diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index a1df07d7e..8b223e769 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -9,7 +9,7 @@ import ( "io" "mime" "mime/multipart" - "net" + "sync" "net/http" "net/textproto" "path/filepath" @@ -21,6 +21,8 @@ import ( "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/util" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" + util_http_client "github.com/seaweedfs/seaweedfs/weed/util/http/client" ) type UploadOption struct { @@ -62,29 +64,47 @@ func (uploadResult *UploadResult) ToPbFileChunk(fileId string, offset int64, tsN } } +var ( + fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", "") + uploader *Uploader + uploaderErr error + once sync.Once +) + // HTTPClient interface for testing type HTTPClient interface { Do(req *http.Request) (*http.Response, error) } -var ( - HttpClient HTTPClient -) +// Uploader +type Uploader struct { + httpClient HTTPClient +} -func init() { - HttpClient = &http.Client{Transport: &http.Transport{ - DialContext: (&net.Dialer{ - Timeout: 10 * time.Second, - KeepAlive: 10 * time.Second, - }).DialContext, - MaxIdleConns: 1024, - MaxIdleConnsPerHost: 1024, - }} +func NewUploader() (*Uploader, error) { + once.Do(func () { + // With Dial context + var httpClient *util_http_client.HTTPClient + httpClient, uploaderErr = util_http.NewGlobalHttpClient(util_http_client.AddDialContext) + if uploaderErr != nil { + uploaderErr = fmt.Errorf("error initializing the loader: %s", uploaderErr) + } + if httpClient != nil { + uploader = newUploader(httpClient) + } + }) + return uploader, uploaderErr +} + +func newUploader(httpClient HTTPClient) (*Uploader) { + return &Uploader{ + httpClient: httpClient, + } } // UploadWithRetry will retry both assigning volume request and uploading content // The option parameter does not need to specify UploadUrl and Jwt, which will come from assigning volume. -func UploadWithRetry(filerClient filer_pb.FilerClient, assignRequest *filer_pb.AssignVolumeRequest, uploadOption *UploadOption, genFileUrlFn func(host, fileId string) string, reader io.Reader) (fileId string, uploadResult *UploadResult, err error, data []byte) { +func (uploader *Uploader) UploadWithRetry(filerClient filer_pb.FilerClient, assignRequest *filer_pb.AssignVolumeRequest, uploadOption *UploadOption, genFileUrlFn func(host, fileId string) string, reader io.Reader) (fileId string, uploadResult *UploadResult, err error, data []byte) { doUploadFunc := func() error { var host string @@ -114,7 +134,7 @@ func UploadWithRetry(filerClient filer_pb.FilerClient, assignRequest *filer_pb.A uploadOption.Jwt = auth var uploadErr error - uploadResult, uploadErr, data = doUpload(reader, uploadOption) + uploadResult, uploadErr, data = uploader.doUpload(reader, uploadOption) return uploadErr } if uploadOption.RetryForever { @@ -130,21 +150,19 @@ func UploadWithRetry(filerClient filer_pb.FilerClient, assignRequest *filer_pb.A return } -var fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", "") - // Upload sends a POST request to a volume server to upload the content with adjustable compression level -func UploadData(data []byte, option *UploadOption) (uploadResult *UploadResult, err error) { - uploadResult, err = retriedUploadData(data, option) +func (uploader *Uploader) UploadData(data []byte, option *UploadOption) (uploadResult *UploadResult, err error) { + uploadResult, err = uploader.retriedUploadData(data, option) return } // Upload sends a POST request to a volume server to upload the content with fast compression -func Upload(reader io.Reader, option *UploadOption) (uploadResult *UploadResult, err error, data []byte) { - uploadResult, err, data = doUpload(reader, option) +func (uploader *Uploader) Upload(reader io.Reader, option *UploadOption) (uploadResult *UploadResult, err error, data []byte) { + uploadResult, err, data = uploader.doUpload(reader, option) return } -func doUpload(reader io.Reader, option *UploadOption) (uploadResult *UploadResult, err error, data []byte) { +func (uploader *Uploader) doUpload(reader io.Reader, option *UploadOption) (uploadResult *UploadResult, err error, data []byte) { bytesReader, ok := reader.(*util.BytesReader) if ok { data = bytesReader.Bytes @@ -155,16 +173,16 @@ func doUpload(reader io.Reader, option *UploadOption) (uploadResult *UploadResul return } } - uploadResult, uploadErr := retriedUploadData(data, option) + uploadResult, uploadErr := uploader.retriedUploadData(data, option) return uploadResult, uploadErr, data } -func retriedUploadData(data []byte, option *UploadOption) (uploadResult *UploadResult, err error) { +func (uploader *Uploader) retriedUploadData(data []byte, option *UploadOption) (uploadResult *UploadResult, err error) { for i := 0; i < 3; i++ { if i > 0 { time.Sleep(time.Millisecond * time.Duration(237*(i+1))) } - uploadResult, err = doUploadData(data, option) + uploadResult, err = uploader.doUploadData(data, option) if err == nil { uploadResult.RetryCount = i return @@ -174,7 +192,7 @@ func retriedUploadData(data []byte, option *UploadOption) (uploadResult *UploadR return } -func doUploadData(data []byte, option *UploadOption) (uploadResult *UploadResult, err error) { +func (uploader *Uploader) doUploadData(data []byte, option *UploadOption) (uploadResult *UploadResult, err error) { contentIsGzipped := option.IsInputCompressed shouldGzipNow := false if !option.IsInputCompressed { @@ -230,7 +248,7 @@ func doUploadData(data []byte, option *UploadOption) (uploadResult *UploadResult } // upload data - uploadResult, err = upload_content(func(w io.Writer) (err error) { + uploadResult, err = uploader.upload_content(func(w io.Writer) (err error) { _, err = w.Write(encryptedData) return }, len(encryptedData), &UploadOption{ @@ -251,7 +269,7 @@ func doUploadData(data []byte, option *UploadOption) (uploadResult *UploadResult uploadResult.Size = uint32(clearDataLen) } else { // upload data - uploadResult, err = upload_content(func(w io.Writer) (err error) { + uploadResult, err = uploader.upload_content(func(w io.Writer) (err error) { _, err = w.Write(data) return }, len(data), &UploadOption{ @@ -277,7 +295,7 @@ func doUploadData(data []byte, option *UploadOption) (uploadResult *UploadResult return uploadResult, err } -func upload_content(fillBufferFunction func(w io.Writer) error, originalDataSize int, option *UploadOption) (*UploadResult, error) { +func (uploader *Uploader) upload_content(fillBufferFunction func(w io.Writer) error, originalDataSize int, option *UploadOption) (*UploadResult, error) { var body_writer *multipart.Writer var reqReader *bytes.Reader var buf *bytebufferpool.ByteBuffer @@ -325,7 +343,7 @@ func upload_content(fillBufferFunction func(w io.Writer) error, originalDataSize } else { reqReader = bytes.NewReader(option.BytesBuffer.Bytes()) } - req, postErr := http.NewRequest("POST", option.UploadUrl, reqReader) + req, postErr := http.NewRequest(http.MethodPost, option.UploadUrl, reqReader) if postErr != nil { glog.V(1).Infof("create upload request %s: %v", option.UploadUrl, postErr) return nil, fmt.Errorf("create upload request %s: %v", option.UploadUrl, postErr) @@ -338,15 +356,15 @@ func upload_content(fillBufferFunction func(w io.Writer) error, originalDataSize req.Header.Set("Authorization", "BEARER "+string(option.Jwt)) } // print("+") - resp, post_err := HttpClient.Do(req) - defer util.CloseResponse(resp) + resp, post_err := uploader.httpClient.Do(req) + defer util_http.CloseResponse(resp) if post_err != nil { if strings.Contains(post_err.Error(), "connection reset by peer") || strings.Contains(post_err.Error(), "use of closed network connection") { glog.V(1).Infof("repeat error upload request %s: %v", option.UploadUrl, postErr) stats.FilerHandlerCounter.WithLabelValues(stats.RepeatErrorUploadContent).Inc() - resp, post_err = HttpClient.Do(req) - defer util.CloseResponse(resp) + resp, post_err = uploader.httpClient.Do(req) + defer util_http.CloseResponse(resp) } } if post_err != nil { diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto index b0829163c..7e948f0dc 100644 --- a/weed/pb/filer.proto +++ b/weed/pb/filer.proto @@ -54,6 +54,9 @@ service SeaweedFiler { rpc GetFilerConfiguration (GetFilerConfigurationRequest) returns (GetFilerConfigurationResponse) { } + rpc TraverseBfsMetadata (TraverseBfsMetadataRequest) returns (stream TraverseBfsMetadataResponse) { + } + rpc SubscribeMetadata (SubscribeMetadataRequest) returns (stream SubscribeMetadataResponse) { } @@ -218,6 +221,7 @@ message DeleteEntryRequest { bool ignore_recursive_error = 6; bool is_from_other_cluster = 7; repeated int32 signatures = 8; + int64 if_not_modified_after = 9; } message DeleteEntryResponse { @@ -341,6 +345,8 @@ message GetFilerConfigurationResponse { string version = 11; string cluster_id = 12; string filer_group = 13; + int32 major_version = 14; + int32 minor_version = 15; } message SubscribeMetadataRequest { @@ -360,6 +366,15 @@ message SubscribeMetadataResponse { int64 ts_ns = 3; } +message TraverseBfsMetadataRequest { + string directory = 1; + repeated string excluded_prefixes = 2; +} +message TraverseBfsMetadataResponse { + string directory = 1; + Entry entry = 2; +} + message LogEntry { int64 ts_ns = 1; int32 partition_key_hash = 2; diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go index 617c13c55..93e4ffd8a 100644 --- a/weed/pb/filer_pb/filer.pb.go +++ b/weed/pb/filer_pb/filer.pb.go @@ -1333,6 +1333,7 @@ type DeleteEntryRequest struct { IgnoreRecursiveError bool `protobuf:"varint,6,opt,name=ignore_recursive_error,json=ignoreRecursiveError,proto3" json:"ignore_recursive_error,omitempty"` IsFromOtherCluster bool `protobuf:"varint,7,opt,name=is_from_other_cluster,json=isFromOtherCluster,proto3" json:"is_from_other_cluster,omitempty"` Signatures []int32 `protobuf:"varint,8,rep,packed,name=signatures,proto3" json:"signatures,omitempty"` + IfNotModifiedAfter int64 `protobuf:"varint,9,opt,name=if_not_modified_after,json=ifNotModifiedAfter,proto3" json:"if_not_modified_after,omitempty"` } func (x *DeleteEntryRequest) Reset() { @@ -1416,6 +1417,13 @@ func (x *DeleteEntryRequest) GetSignatures() []int32 { return nil } +func (x *DeleteEntryRequest) GetIfNotModifiedAfter() int64 { + if x != nil { + return x.IfNotModifiedAfter + } + return 0 +} + type DeleteEntryResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2681,6 +2689,8 @@ type GetFilerConfigurationResponse struct { Version string `protobuf:"bytes,11,opt,name=version,proto3" json:"version,omitempty"` ClusterId string `protobuf:"bytes,12,opt,name=cluster_id,json=clusterId,proto3" json:"cluster_id,omitempty"` FilerGroup string `protobuf:"bytes,13,opt,name=filer_group,json=filerGroup,proto3" json:"filer_group,omitempty"` + MajorVersion int32 `protobuf:"varint,14,opt,name=major_version,json=majorVersion,proto3" json:"major_version,omitempty"` + MinorVersion int32 `protobuf:"varint,15,opt,name=minor_version,json=minorVersion,proto3" json:"minor_version,omitempty"` } func (x *GetFilerConfigurationResponse) Reset() { @@ -2799,6 +2809,20 @@ func (x *GetFilerConfigurationResponse) GetFilerGroup() string { return "" } +func (x *GetFilerConfigurationResponse) GetMajorVersion() int32 { + if x != nil { + return x.MajorVersion + } + return 0 +} + +func (x *GetFilerConfigurationResponse) GetMinorVersion() int32 { + if x != nil { + return x.MinorVersion + } + return 0 +} + type SubscribeMetadataRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2973,6 +2997,116 @@ func (x *SubscribeMetadataResponse) GetTsNs() int64 { return 0 } +type TraverseBfsMetadataRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Directory string `protobuf:"bytes,1,opt,name=directory,proto3" json:"directory,omitempty"` + ExcludedPrefixes []string `protobuf:"bytes,2,rep,name=excluded_prefixes,json=excludedPrefixes,proto3" json:"excluded_prefixes,omitempty"` +} + +func (x *TraverseBfsMetadataRequest) Reset() { + *x = TraverseBfsMetadataRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_filer_proto_msgTypes[43] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TraverseBfsMetadataRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TraverseBfsMetadataRequest) ProtoMessage() {} + +func (x *TraverseBfsMetadataRequest) ProtoReflect() protoreflect.Message { + mi := &file_filer_proto_msgTypes[43] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TraverseBfsMetadataRequest.ProtoReflect.Descriptor instead. +func (*TraverseBfsMetadataRequest) Descriptor() ([]byte, []int) { + return file_filer_proto_rawDescGZIP(), []int{43} +} + +func (x *TraverseBfsMetadataRequest) GetDirectory() string { + if x != nil { + return x.Directory + } + return "" +} + +func (x *TraverseBfsMetadataRequest) GetExcludedPrefixes() []string { + if x != nil { + return x.ExcludedPrefixes + } + return nil +} + +type TraverseBfsMetadataResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Directory string `protobuf:"bytes,1,opt,name=directory,proto3" json:"directory,omitempty"` + Entry *Entry `protobuf:"bytes,2,opt,name=entry,proto3" json:"entry,omitempty"` +} + +func (x *TraverseBfsMetadataResponse) Reset() { + *x = TraverseBfsMetadataResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_filer_proto_msgTypes[44] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TraverseBfsMetadataResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TraverseBfsMetadataResponse) ProtoMessage() {} + +func (x *TraverseBfsMetadataResponse) ProtoReflect() protoreflect.Message { + mi := &file_filer_proto_msgTypes[44] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TraverseBfsMetadataResponse.ProtoReflect.Descriptor instead. +func (*TraverseBfsMetadataResponse) Descriptor() ([]byte, []int) { + return file_filer_proto_rawDescGZIP(), []int{44} +} + +func (x *TraverseBfsMetadataResponse) GetDirectory() string { + if x != nil { + return x.Directory + } + return "" +} + +func (x *TraverseBfsMetadataResponse) GetEntry() *Entry { + if x != nil { + return x.Entry + } + return nil +} + type LogEntry struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2987,7 +3121,7 @@ type LogEntry struct { func (x *LogEntry) Reset() { *x = LogEntry{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[43] + mi := &file_filer_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3000,7 +3134,7 @@ func (x *LogEntry) String() string { func (*LogEntry) ProtoMessage() {} func (x *LogEntry) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[43] + mi := &file_filer_proto_msgTypes[45] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3013,7 +3147,7 @@ func (x *LogEntry) ProtoReflect() protoreflect.Message { // Deprecated: Use LogEntry.ProtoReflect.Descriptor instead. func (*LogEntry) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{43} + return file_filer_proto_rawDescGZIP(), []int{45} } func (x *LogEntry) GetTsNs() int64 { @@ -3057,7 +3191,7 @@ type KeepConnectedRequest struct { func (x *KeepConnectedRequest) Reset() { *x = KeepConnectedRequest{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[44] + mi := &file_filer_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3070,7 +3204,7 @@ func (x *KeepConnectedRequest) String() string { func (*KeepConnectedRequest) ProtoMessage() {} func (x *KeepConnectedRequest) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[44] + mi := &file_filer_proto_msgTypes[46] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3083,7 +3217,7 @@ func (x *KeepConnectedRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use KeepConnectedRequest.ProtoReflect.Descriptor instead. func (*KeepConnectedRequest) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{44} + return file_filer_proto_rawDescGZIP(), []int{46} } func (x *KeepConnectedRequest) GetName() string { @@ -3116,7 +3250,7 @@ type KeepConnectedResponse struct { func (x *KeepConnectedResponse) Reset() { *x = KeepConnectedResponse{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[45] + mi := &file_filer_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3129,7 +3263,7 @@ func (x *KeepConnectedResponse) String() string { func (*KeepConnectedResponse) ProtoMessage() {} func (x *KeepConnectedResponse) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[45] + mi := &file_filer_proto_msgTypes[47] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3142,7 +3276,7 @@ func (x *KeepConnectedResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use KeepConnectedResponse.ProtoReflect.Descriptor instead. func (*KeepConnectedResponse) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{45} + return file_filer_proto_rawDescGZIP(), []int{47} } type LocateBrokerRequest struct { @@ -3156,7 +3290,7 @@ type LocateBrokerRequest struct { func (x *LocateBrokerRequest) Reset() { *x = LocateBrokerRequest{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[46] + mi := &file_filer_proto_msgTypes[48] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3169,7 +3303,7 @@ func (x *LocateBrokerRequest) String() string { func (*LocateBrokerRequest) ProtoMessage() {} func (x *LocateBrokerRequest) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[46] + mi := &file_filer_proto_msgTypes[48] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3182,7 +3316,7 @@ func (x *LocateBrokerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use LocateBrokerRequest.ProtoReflect.Descriptor instead. func (*LocateBrokerRequest) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{46} + return file_filer_proto_rawDescGZIP(), []int{48} } func (x *LocateBrokerRequest) GetResource() string { @@ -3204,7 +3338,7 @@ type LocateBrokerResponse struct { func (x *LocateBrokerResponse) Reset() { *x = LocateBrokerResponse{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[47] + mi := &file_filer_proto_msgTypes[49] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3217,7 +3351,7 @@ func (x *LocateBrokerResponse) String() string { func (*LocateBrokerResponse) ProtoMessage() {} func (x *LocateBrokerResponse) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[47] + mi := &file_filer_proto_msgTypes[49] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3230,7 +3364,7 @@ func (x *LocateBrokerResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use LocateBrokerResponse.ProtoReflect.Descriptor instead. func (*LocateBrokerResponse) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{47} + return file_filer_proto_rawDescGZIP(), []int{49} } func (x *LocateBrokerResponse) GetFound() bool { @@ -3261,7 +3395,7 @@ type KvGetRequest struct { func (x *KvGetRequest) Reset() { *x = KvGetRequest{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[48] + mi := &file_filer_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3274,7 +3408,7 @@ func (x *KvGetRequest) String() string { func (*KvGetRequest) ProtoMessage() {} func (x *KvGetRequest) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[48] + mi := &file_filer_proto_msgTypes[50] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3287,7 +3421,7 @@ func (x *KvGetRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use KvGetRequest.ProtoReflect.Descriptor instead. func (*KvGetRequest) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{48} + return file_filer_proto_rawDescGZIP(), []int{50} } func (x *KvGetRequest) GetKey() []byte { @@ -3309,7 +3443,7 @@ type KvGetResponse struct { func (x *KvGetResponse) Reset() { *x = KvGetResponse{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[49] + mi := &file_filer_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3322,7 +3456,7 @@ func (x *KvGetResponse) String() string { func (*KvGetResponse) ProtoMessage() {} func (x *KvGetResponse) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[49] + mi := &file_filer_proto_msgTypes[51] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3335,7 +3469,7 @@ func (x *KvGetResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use KvGetResponse.ProtoReflect.Descriptor instead. func (*KvGetResponse) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{49} + return file_filer_proto_rawDescGZIP(), []int{51} } func (x *KvGetResponse) GetValue() []byte { @@ -3364,7 +3498,7 @@ type KvPutRequest struct { func (x *KvPutRequest) Reset() { *x = KvPutRequest{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[50] + mi := &file_filer_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3377,7 +3511,7 @@ func (x *KvPutRequest) String() string { func (*KvPutRequest) ProtoMessage() {} func (x *KvPutRequest) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[50] + mi := &file_filer_proto_msgTypes[52] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3390,7 +3524,7 @@ func (x *KvPutRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use KvPutRequest.ProtoReflect.Descriptor instead. func (*KvPutRequest) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{50} + return file_filer_proto_rawDescGZIP(), []int{52} } func (x *KvPutRequest) GetKey() []byte { @@ -3418,7 +3552,7 @@ type KvPutResponse struct { func (x *KvPutResponse) Reset() { *x = KvPutResponse{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[51] + mi := &file_filer_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3431,7 +3565,7 @@ func (x *KvPutResponse) String() string { func (*KvPutResponse) ProtoMessage() {} func (x *KvPutResponse) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[51] + mi := &file_filer_proto_msgTypes[53] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3444,7 +3578,7 @@ func (x *KvPutResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use KvPutResponse.ProtoReflect.Descriptor instead. func (*KvPutResponse) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{51} + return file_filer_proto_rawDescGZIP(), []int{53} } func (x *KvPutResponse) GetError() string { @@ -3469,7 +3603,7 @@ type FilerConf struct { func (x *FilerConf) Reset() { *x = FilerConf{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[52] + mi := &file_filer_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3482,7 +3616,7 @@ func (x *FilerConf) String() string { func (*FilerConf) ProtoMessage() {} func (x *FilerConf) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[52] + mi := &file_filer_proto_msgTypes[54] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3495,7 +3629,7 @@ func (x *FilerConf) ProtoReflect() protoreflect.Message { // Deprecated: Use FilerConf.ProtoReflect.Descriptor instead. func (*FilerConf) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{52} + return file_filer_proto_rawDescGZIP(), []int{54} } func (x *FilerConf) GetVersion() int32 { @@ -3527,7 +3661,7 @@ type CacheRemoteObjectToLocalClusterRequest struct { func (x *CacheRemoteObjectToLocalClusterRequest) Reset() { *x = CacheRemoteObjectToLocalClusterRequest{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[53] + mi := &file_filer_proto_msgTypes[55] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3540,7 +3674,7 @@ func (x *CacheRemoteObjectToLocalClusterRequest) String() string { func (*CacheRemoteObjectToLocalClusterRequest) ProtoMessage() {} func (x *CacheRemoteObjectToLocalClusterRequest) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[53] + mi := &file_filer_proto_msgTypes[55] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3553,7 +3687,7 @@ func (x *CacheRemoteObjectToLocalClusterRequest) ProtoReflect() protoreflect.Mes // Deprecated: Use CacheRemoteObjectToLocalClusterRequest.ProtoReflect.Descriptor instead. func (*CacheRemoteObjectToLocalClusterRequest) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{53} + return file_filer_proto_rawDescGZIP(), []int{55} } func (x *CacheRemoteObjectToLocalClusterRequest) GetDirectory() string { @@ -3581,7 +3715,7 @@ type CacheRemoteObjectToLocalClusterResponse struct { func (x *CacheRemoteObjectToLocalClusterResponse) Reset() { *x = CacheRemoteObjectToLocalClusterResponse{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[54] + mi := &file_filer_proto_msgTypes[56] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3594,7 +3728,7 @@ func (x *CacheRemoteObjectToLocalClusterResponse) String() string { func (*CacheRemoteObjectToLocalClusterResponse) ProtoMessage() {} func (x *CacheRemoteObjectToLocalClusterResponse) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[54] + mi := &file_filer_proto_msgTypes[56] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3607,7 +3741,7 @@ func (x *CacheRemoteObjectToLocalClusterResponse) ProtoReflect() protoreflect.Me // Deprecated: Use CacheRemoteObjectToLocalClusterResponse.ProtoReflect.Descriptor instead. func (*CacheRemoteObjectToLocalClusterResponse) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{54} + return file_filer_proto_rawDescGZIP(), []int{56} } func (x *CacheRemoteObjectToLocalClusterResponse) GetEntry() *Entry { @@ -3635,7 +3769,7 @@ type LockRequest struct { func (x *LockRequest) Reset() { *x = LockRequest{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[55] + mi := &file_filer_proto_msgTypes[57] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3648,7 +3782,7 @@ func (x *LockRequest) String() string { func (*LockRequest) ProtoMessage() {} func (x *LockRequest) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[55] + mi := &file_filer_proto_msgTypes[57] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3661,7 +3795,7 @@ func (x *LockRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use LockRequest.ProtoReflect.Descriptor instead. func (*LockRequest) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{55} + return file_filer_proto_rawDescGZIP(), []int{57} } func (x *LockRequest) GetName() string { @@ -3713,7 +3847,7 @@ type LockResponse struct { func (x *LockResponse) Reset() { *x = LockResponse{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[56] + mi := &file_filer_proto_msgTypes[58] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3726,7 +3860,7 @@ func (x *LockResponse) String() string { func (*LockResponse) ProtoMessage() {} func (x *LockResponse) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[56] + mi := &file_filer_proto_msgTypes[58] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3739,7 +3873,7 @@ func (x *LockResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use LockResponse.ProtoReflect.Descriptor instead. func (*LockResponse) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{56} + return file_filer_proto_rawDescGZIP(), []int{58} } func (x *LockResponse) GetRenewToken() string { @@ -3783,7 +3917,7 @@ type UnlockRequest struct { func (x *UnlockRequest) Reset() { *x = UnlockRequest{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[57] + mi := &file_filer_proto_msgTypes[59] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3796,7 +3930,7 @@ func (x *UnlockRequest) String() string { func (*UnlockRequest) ProtoMessage() {} func (x *UnlockRequest) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[57] + mi := &file_filer_proto_msgTypes[59] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3809,7 +3943,7 @@ func (x *UnlockRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UnlockRequest.ProtoReflect.Descriptor instead. func (*UnlockRequest) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{57} + return file_filer_proto_rawDescGZIP(), []int{59} } func (x *UnlockRequest) GetName() string { @@ -3845,7 +3979,7 @@ type UnlockResponse struct { func (x *UnlockResponse) Reset() { *x = UnlockResponse{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[58] + mi := &file_filer_proto_msgTypes[60] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3858,7 +3992,7 @@ func (x *UnlockResponse) String() string { func (*UnlockResponse) ProtoMessage() {} func (x *UnlockResponse) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[58] + mi := &file_filer_proto_msgTypes[60] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3871,7 +4005,7 @@ func (x *UnlockResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use UnlockResponse.ProtoReflect.Descriptor instead. func (*UnlockResponse) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{58} + return file_filer_proto_rawDescGZIP(), []int{60} } func (x *UnlockResponse) GetError() string { @@ -3900,7 +4034,7 @@ type FindLockOwnerRequest struct { func (x *FindLockOwnerRequest) Reset() { *x = FindLockOwnerRequest{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[59] + mi := &file_filer_proto_msgTypes[61] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3913,7 +4047,7 @@ func (x *FindLockOwnerRequest) String() string { func (*FindLockOwnerRequest) ProtoMessage() {} func (x *FindLockOwnerRequest) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[59] + mi := &file_filer_proto_msgTypes[61] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3926,7 +4060,7 @@ func (x *FindLockOwnerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FindLockOwnerRequest.ProtoReflect.Descriptor instead. func (*FindLockOwnerRequest) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{59} + return file_filer_proto_rawDescGZIP(), []int{61} } func (x *FindLockOwnerRequest) GetName() string { @@ -3954,7 +4088,7 @@ type FindLockOwnerResponse struct { func (x *FindLockOwnerResponse) Reset() { *x = FindLockOwnerResponse{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[60] + mi := &file_filer_proto_msgTypes[62] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3967,7 +4101,7 @@ func (x *FindLockOwnerResponse) String() string { func (*FindLockOwnerResponse) ProtoMessage() {} func (x *FindLockOwnerResponse) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[60] + mi := &file_filer_proto_msgTypes[62] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3980,7 +4114,7 @@ func (x *FindLockOwnerResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use FindLockOwnerResponse.ProtoReflect.Descriptor instead. func (*FindLockOwnerResponse) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{60} + return file_filer_proto_rawDescGZIP(), []int{62} } func (x *FindLockOwnerResponse) GetOwner() string { @@ -4004,7 +4138,7 @@ type Lock struct { func (x *Lock) Reset() { *x = Lock{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[61] + mi := &file_filer_proto_msgTypes[63] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4017,7 +4151,7 @@ func (x *Lock) String() string { func (*Lock) ProtoMessage() {} func (x *Lock) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[61] + mi := &file_filer_proto_msgTypes[63] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4030,7 +4164,7 @@ func (x *Lock) ProtoReflect() protoreflect.Message { // Deprecated: Use Lock.ProtoReflect.Descriptor instead. func (*Lock) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{61} + return file_filer_proto_rawDescGZIP(), []int{63} } func (x *Lock) GetName() string { @@ -4072,7 +4206,7 @@ type TransferLocksRequest struct { func (x *TransferLocksRequest) Reset() { *x = TransferLocksRequest{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[62] + mi := &file_filer_proto_msgTypes[64] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4085,7 +4219,7 @@ func (x *TransferLocksRequest) String() string { func (*TransferLocksRequest) ProtoMessage() {} func (x *TransferLocksRequest) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[62] + mi := &file_filer_proto_msgTypes[64] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4098,7 +4232,7 @@ func (x *TransferLocksRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use TransferLocksRequest.ProtoReflect.Descriptor instead. func (*TransferLocksRequest) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{62} + return file_filer_proto_rawDescGZIP(), []int{64} } func (x *TransferLocksRequest) GetLocks() []*Lock { @@ -4117,7 +4251,7 @@ type TransferLocksResponse struct { func (x *TransferLocksResponse) Reset() { *x = TransferLocksResponse{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[63] + mi := &file_filer_proto_msgTypes[65] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4130,7 +4264,7 @@ func (x *TransferLocksResponse) String() string { func (*TransferLocksResponse) ProtoMessage() {} func (x *TransferLocksResponse) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[63] + mi := &file_filer_proto_msgTypes[65] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4143,7 +4277,7 @@ func (x *TransferLocksResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use TransferLocksResponse.ProtoReflect.Descriptor instead. func (*TransferLocksResponse) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{63} + return file_filer_proto_rawDescGZIP(), []int{65} } // if found, send the exact address @@ -4160,7 +4294,7 @@ type LocateBrokerResponse_Resource struct { func (x *LocateBrokerResponse_Resource) Reset() { *x = LocateBrokerResponse_Resource{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[66] + mi := &file_filer_proto_msgTypes[68] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4173,7 +4307,7 @@ func (x *LocateBrokerResponse_Resource) String() string { func (*LocateBrokerResponse_Resource) ProtoMessage() {} func (x *LocateBrokerResponse_Resource) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[66] + mi := &file_filer_proto_msgTypes[68] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4186,7 +4320,7 @@ func (x *LocateBrokerResponse_Resource) ProtoReflect() protoreflect.Message { // Deprecated: Use LocateBrokerResponse_Resource.ProtoReflect.Descriptor instead. func (*LocateBrokerResponse_Resource) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{47, 0} + return file_filer_proto_rawDescGZIP(), []int{49, 0} } func (x *LocateBrokerResponse_Resource) GetGrpcAddresses() string { @@ -4226,7 +4360,7 @@ type FilerConf_PathConf struct { func (x *FilerConf_PathConf) Reset() { *x = FilerConf_PathConf{} if protoimpl.UnsafeEnabled { - mi := &file_filer_proto_msgTypes[67] + mi := &file_filer_proto_msgTypes[69] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4239,7 +4373,7 @@ func (x *FilerConf_PathConf) String() string { func (*FilerConf_PathConf) ProtoMessage() {} func (x *FilerConf_PathConf) ProtoReflect() protoreflect.Message { - mi := &file_filer_proto_msgTypes[67] + mi := &file_filer_proto_msgTypes[69] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4252,7 +4386,7 @@ func (x *FilerConf_PathConf) ProtoReflect() protoreflect.Message { // Deprecated: Use FilerConf_PathConf.ProtoReflect.Descriptor instead. func (*FilerConf_PathConf) Descriptor() ([]byte, []int) { - return file_filer_proto_rawDescGZIP(), []int{52, 0} + return file_filer_proto_rawDescGZIP(), []int{54, 0} } func (x *FilerConf_PathConf) GetLocationPrefix() string { @@ -4536,7 +4670,7 @@ var file_filer_proto_rawDesc = []byte{ 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x52, 0x06, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x22, 0x17, 0x0a, 0x15, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x54, 0x6f, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x98, 0x02, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xcb, 0x02, 0x0a, 0x12, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, @@ -4554,482 +4688,509 @@ var file_filer_proto_rawDesc = []byte{ 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x69, 0x73, 0x46, 0x72, 0x6f, 0x6d, 0x4f, 0x74, 0x68, 0x65, 0x72, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x05, 0x52, 0x0a, 0x73, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x22, 0x2b, 0x0a, 0x13, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xba, 0x01, 0x0a, 0x18, 0x41, 0x74, 0x6f, 0x6d, 0x69, 0x63, - 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x6f, 0x6c, 0x64, 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, - 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6f, 0x6c, 0x64, 0x44, 0x69, - 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x6f, 0x6c, 0x64, 0x5f, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6f, 0x6c, 0x64, 0x4e, 0x61, - 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x6e, 0x65, 0x77, 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, - 0x6f, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6e, 0x65, 0x77, 0x44, 0x69, - 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x65, 0x77, 0x5f, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x65, 0x77, 0x4e, 0x61, - 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, - 0x18, 0x05, 0x20, 0x03, 0x28, 0x05, 0x52, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x73, 0x22, 0x1b, 0x0a, 0x19, 0x41, 0x74, 0x6f, 0x6d, 0x69, 0x63, 0x52, 0x65, 0x6e, 0x61, - 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0xba, 0x01, 0x0a, 0x18, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x23, 0x0a, 0x0d, - 0x6f, 0x6c, 0x64, 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6f, 0x6c, 0x64, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, - 0x79, 0x12, 0x19, 0x0a, 0x08, 0x6f, 0x6c, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x6f, 0x6c, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x0d, - 0x6e, 0x65, 0x77, 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6e, 0x65, 0x77, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, - 0x79, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x65, 0x77, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x65, 0x77, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x05, - 0x52, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x22, 0x9a, 0x01, 0x0a, - 0x19, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x69, - 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, - 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x4a, 0x0a, 0x12, 0x65, 0x76, 0x65, 0x6e, - 0x74, 0x5f, 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x11, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x13, 0x0a, 0x05, 0x74, 0x73, 0x5f, 0x6e, 0x73, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x04, 0x74, 0x73, 0x4e, 0x73, 0x22, 0x89, 0x02, 0x0a, 0x13, 0x41, 0x73, - 0x73, 0x69, 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, - 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, - 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, - 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x74, 0x6c, - 0x5f, 0x73, 0x65, 0x63, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x74, 0x74, 0x6c, 0x53, - 0x65, 0x63, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x63, 0x65, 0x6e, 0x74, 0x65, - 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, - 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x61, 0x63, 0x6b, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x61, 0x63, 0x6b, 0x12, 0x1b, 0x0a, 0x09, 0x64, - 0x61, 0x74, 0x61, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, - 0x64, 0x61, 0x74, 0x61, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, - 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, - 0x6b, 0x54, 0x79, 0x70, 0x65, 0x22, 0xe1, 0x01, 0x0a, 0x14, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, - 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x17, - 0x0a, 0x07, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x66, 0x69, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x12, 0x0a, - 0x04, 0x61, 0x75, 0x74, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x61, 0x75, 0x74, - 0x68, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x08, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x2e, 0x0a, 0x08, 0x6c, 0x6f, 0x63, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x69, - 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x34, 0x0a, 0x13, 0x4c, 0x6f, 0x6f, - 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x1d, 0x0a, 0x0a, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x73, 0x22, - 0x3d, 0x0a, 0x09, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x30, 0x0a, 0x09, - 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x12, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x79, - 0x0a, 0x08, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, - 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, - 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x55, 0x72, 0x6c, 0x12, 0x1b, 0x0a, 0x09, 0x67, - 0x72, 0x70, 0x63, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, - 0x67, 0x72, 0x70, 0x63, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, - 0x5f, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, - 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x22, 0xc3, 0x01, 0x0a, 0x14, 0x4c, 0x6f, - 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x55, 0x0a, 0x0d, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x5f, - 0x6d, 0x61, 0x70, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x66, 0x69, 0x6c, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x6c, 0x6f, 0x63, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x4d, 0x61, 0x70, 0x1a, 0x54, 0x0a, 0x11, 0x4c, 0x6f, 0x63, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x12, 0x29, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x13, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, - 0x20, 0x0a, 0x0a, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, - 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x22, 0x7b, 0x0a, 0x15, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, - 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x34, 0x0a, 0x16, 0x69, 0x6e, - 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x6e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x5f, 0x76, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x69, 0x6e, 0x63, 0x6c, - 0x75, 0x64, 0x65, 0x4e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x73, - 0x12, 0x2c, 0x0a, 0x12, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x65, 0x63, 0x5f, 0x76, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x69, 0x6e, - 0x63, 0x6c, 0x75, 0x64, 0x65, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x73, 0x22, 0x50, - 0x0a, 0x16, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x0b, 0x63, 0x6f, 0x6c, 0x6c, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, - 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x0b, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x22, 0x39, 0x0a, 0x17, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x63, - 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x1a, 0x0a, 0x18, 0x44, - 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x84, 0x01, 0x0a, 0x11, 0x53, 0x74, 0x61, 0x74, - 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x20, 0x0a, - 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x10, 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, 0x74, - 0x6c, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x22, 0x6f, - 0x0a, 0x12, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x73, 0x69, - 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x53, - 0x69, 0x7a, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x75, 0x73, 0x65, 0x64, 0x5f, 0x73, 0x69, 0x7a, 0x65, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x75, 0x73, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65, - 0x12, 0x1d, 0x0a, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, - 0x46, 0x0a, 0x0b, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, - 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, - 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x74, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0x7a, 0x0a, 0x0c, 0x50, 0x69, 0x6e, 0x67, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x22, 0x0a, 0x0d, 0x73, 0x74, 0x61, 0x72, 0x74, - 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, - 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x4e, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x72, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6e, 0x73, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x0c, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x4e, - 0x73, 0x12, 0x20, 0x0a, 0x0c, 0x73, 0x74, 0x6f, 0x70, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6e, - 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x73, 0x74, 0x6f, 0x70, 0x54, 0x69, 0x6d, - 0x65, 0x4e, 0x73, 0x22, 0x1e, 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x22, 0x9e, 0x03, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x72, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x73, 0x12, - 0x20, 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x15, 0x0a, 0x06, 0x6d, 0x61, 0x78, 0x5f, 0x6d, 0x62, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x0d, 0x52, 0x05, 0x6d, 0x61, 0x78, 0x4d, 0x62, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x69, 0x72, 0x5f, - 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, - 0x69, 0x72, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x69, 0x70, - 0x68, 0x65, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x63, 0x69, 0x70, 0x68, 0x65, - 0x72, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x08, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, - 0x27, 0x0a, 0x0f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, - 0x73, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, - 0x73, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x30, 0x0a, 0x14, 0x6d, 0x65, 0x74, 0x72, - 0x69, 0x63, 0x73, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x63, - 0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, 0x52, 0x12, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x49, - 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x53, 0x65, 0x63, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, - 0x69, 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, - 0x72, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, - 0x75, 0x70, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x47, - 0x72, 0x6f, 0x75, 0x70, 0x22, 0xb7, 0x02, 0x0a, 0x18, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, - 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4e, 0x61, - 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x61, 0x74, 0x68, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, - 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x70, 0x61, 0x74, 0x68, 0x50, 0x72, 0x65, - 0x66, 0x69, 0x78, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x73, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x4e, 0x73, 0x12, 0x1c, - 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x05, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x23, 0x0a, 0x0d, - 0x70, 0x61, 0x74, 0x68, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x18, 0x06, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x0c, 0x70, 0x61, 0x74, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, - 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x07, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x19, - 0x0a, 0x08, 0x75, 0x6e, 0x74, 0x69, 0x6c, 0x5f, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x07, 0x75, 0x6e, 0x74, 0x69, 0x6c, 0x4e, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x69, - 0x65, 0x6e, 0x74, 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x20, 0x0a, 0x0b, - 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x0b, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x22, 0x9a, - 0x01, 0x0a, 0x19, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, - 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x4a, 0x0a, 0x12, 0x65, 0x76, - 0x65, 0x6e, 0x74, 0x5f, 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, - 0x62, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x11, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, - 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x13, 0x0a, 0x05, 0x74, 0x73, 0x5f, 0x6e, 0x73, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x74, 0x73, 0x4e, 0x73, 0x22, 0x73, 0x0a, 0x08, 0x4c, - 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x13, 0x0a, 0x05, 0x74, 0x73, 0x5f, 0x6e, 0x73, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x74, 0x73, 0x4e, 0x73, 0x12, 0x2c, 0x0a, 0x12, - 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x68, 0x61, - 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x10, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, - 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x48, 0x61, 0x73, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, - 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x22, 0x65, 0x0a, 0x14, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, - 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, - 0x67, 0x72, 0x70, 0x63, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, - 0x08, 0x67, 0x72, 0x70, 0x63, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x22, 0x17, 0x0a, 0x15, 0x4b, 0x65, 0x65, 0x70, 0x43, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x31, 0x0a, 0x13, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x22, 0xcd, 0x01, 0x0a, 0x14, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x42, 0x72, - 0x6f, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, - 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, 0x75, - 0x6e, 0x64, 0x12, 0x45, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, - 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, - 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x09, - 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x1a, 0x58, 0x0a, 0x08, 0x52, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x61, 0x64, - 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x67, - 0x72, 0x70, 0x63, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x12, 0x25, 0x0a, 0x0e, - 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x43, 0x6f, - 0x75, 0x6e, 0x74, 0x22, 0x20, 0x0a, 0x0c, 0x4b, 0x76, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x3b, 0x0a, 0x0d, 0x4b, 0x76, 0x47, 0x65, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x14, 0x0a, 0x05, - 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, - 0x6f, 0x72, 0x22, 0x36, 0x0a, 0x0c, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x25, 0x0a, 0x0d, 0x4b, 0x76, - 0x50, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, - 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, - 0x72, 0x22, 0xa4, 0x04, 0x0a, 0x09, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x12, - 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x3a, 0x0a, 0x09, 0x6c, 0x6f, 0x63, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x66, - 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, - 0x66, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0xc0, 0x03, 0x0a, 0x08, 0x50, 0x61, 0x74, 0x68, 0x43, 0x6f, - 0x6e, 0x66, 0x12, 0x27, 0x0a, 0x0f, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, - 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6c, 0x6f, 0x63, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x1e, 0x0a, 0x0a, 0x63, - 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x72, - 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, - 0x03, 0x74, 0x74, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, 0x74, 0x6c, 0x12, - 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, - 0x66, 0x73, 0x79, 0x6e, 0x63, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x73, 0x79, - 0x6e, 0x63, 0x12, 0x2e, 0x0a, 0x13, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x67, 0x72, 0x6f, - 0x77, 0x74, 0x68, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, - 0x11, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x47, 0x72, 0x6f, 0x77, 0x74, 0x68, 0x43, 0x6f, 0x75, - 0x6e, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, - 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x72, 0x65, 0x61, 0x64, 0x4f, 0x6e, 0x6c, 0x79, 0x12, - 0x1f, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x09, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, 0x74, 0x65, 0x72, - 0x12, 0x12, 0x0a, 0x04, 0x72, 0x61, 0x63, 0x6b, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x72, 0x61, 0x63, 0x6b, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x6e, 0x6f, 0x64, - 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x61, 0x74, 0x61, 0x4e, 0x6f, 0x64, - 0x65, 0x12, 0x2f, 0x0a, 0x14, 0x6d, 0x61, 0x78, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6e, 0x61, - 0x6d, 0x65, 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0d, 0x52, - 0x11, 0x6d, 0x61, 0x78, 0x46, 0x69, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x4c, 0x65, 0x6e, 0x67, - 0x74, 0x68, 0x12, 0x34, 0x0a, 0x16, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x63, 0x68, - 0x75, 0x6e, 0x6b, 0x5f, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0d, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x14, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x43, 0x68, 0x75, 0x6e, 0x6b, - 0x44, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x5a, 0x0a, 0x26, 0x43, 0x61, 0x63, 0x68, - 0x65, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x54, 0x6f, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, - 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x50, 0x0a, 0x27, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x6d, - 0x6f, 0x74, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x54, 0x6f, 0x4c, 0x6f, 0x63, 0x61, 0x6c, - 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x25, 0x0a, 0x05, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, - 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x05, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x22, 0x9b, 0x01, 0x0a, 0x0b, 0x4c, 0x6f, 0x63, 0x6b, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x26, 0x0a, 0x0f, 0x73, 0x65, - 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x5f, 0x74, 0x6f, 0x5f, 0x6c, 0x6f, 0x63, 0x6b, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x0d, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x54, 0x6f, 0x4c, 0x6f, - 0x63, 0x6b, 0x12, 0x1f, 0x0a, 0x0b, 0x72, 0x65, 0x6e, 0x65, 0x77, 0x5f, 0x74, 0x6f, 0x6b, 0x65, - 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x6e, 0x65, 0x77, 0x54, 0x6f, - 0x6b, 0x65, 0x6e, 0x12, 0x19, 0x0a, 0x08, 0x69, 0x73, 0x5f, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x69, 0x73, 0x4d, 0x6f, 0x76, 0x65, 0x64, 0x12, 0x14, - 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6f, - 0x77, 0x6e, 0x65, 0x72, 0x22, 0x91, 0x01, 0x0a, 0x0c, 0x4c, 0x6f, 0x63, 0x6b, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x72, 0x65, 0x6e, 0x65, 0x77, 0x5f, 0x74, - 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x6e, 0x65, - 0x77, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6f, - 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x6b, - 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x2b, 0x0a, 0x12, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x68, 0x6f, - 0x73, 0x74, 0x5f, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x5f, 0x74, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0f, 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x6f, 0x73, 0x74, 0x4d, 0x6f, 0x76, 0x65, 0x64, - 0x54, 0x6f, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x5f, 0x0a, 0x0d, 0x55, 0x6e, 0x6c, 0x6f, - 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, - 0x0b, 0x72, 0x65, 0x6e, 0x65, 0x77, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x6e, 0x65, 0x77, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x19, - 0x0a, 0x08, 0x69, 0x73, 0x5f, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x07, 0x69, 0x73, 0x4d, 0x6f, 0x76, 0x65, 0x64, 0x22, 0x41, 0x0a, 0x0e, 0x55, 0x6e, 0x6c, - 0x6f, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, - 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, - 0x72, 0x12, 0x19, 0x0a, 0x08, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x5f, 0x74, 0x6f, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x54, 0x6f, 0x22, 0x45, 0x0a, 0x14, - 0x46, 0x69, 0x6e, 0x64, 0x4c, 0x6f, 0x63, 0x6b, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x69, 0x73, 0x5f, 0x6d, - 0x6f, 0x76, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x69, 0x73, 0x4d, 0x6f, - 0x76, 0x65, 0x64, 0x22, 0x2d, 0x0a, 0x15, 0x46, 0x69, 0x6e, 0x64, 0x4c, 0x6f, 0x63, 0x6b, 0x4f, - 0x77, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, - 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6f, 0x77, 0x6e, - 0x65, 0x72, 0x22, 0x75, 0x0a, 0x04, 0x4c, 0x6f, 0x63, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, - 0x0a, 0x0b, 0x72, 0x65, 0x6e, 0x65, 0x77, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x6e, 0x65, 0x77, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, - 0x22, 0x0a, 0x0d, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x5f, 0x6e, 0x73, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x64, 0x41, - 0x74, 0x4e, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x22, 0x3c, 0x0a, 0x14, 0x54, 0x72, 0x61, - 0x6e, 0x73, 0x66, 0x65, 0x72, 0x4c, 0x6f, 0x63, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x24, 0x0a, 0x05, 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x0e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x6b, - 0x52, 0x05, 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x22, 0x17, 0x0a, 0x15, 0x54, 0x72, 0x61, 0x6e, 0x73, - 0x66, 0x65, 0x72, 0x4c, 0x6f, 0x63, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x32, 0x8f, 0x10, 0x0a, 0x0c, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, - 0x72, 0x12, 0x67, 0x0a, 0x14, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x69, 0x72, 0x65, 0x63, - 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x25, 0x2e, 0x66, 0x69, 0x6c, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x69, 0x72, 0x65, 0x63, - 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x26, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, - 0x75, 0x70, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4e, 0x0a, 0x0b, 0x4c, 0x69, - 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x4c, 0x0a, 0x0b, 0x43, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4c, 0x0a, 0x0b, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, - 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x52, 0x0a, 0x0d, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, - 0x54, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x54, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x54, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4c, 0x0a, 0x0b, 0x44, 0x65, - 0x6c, 0x65, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5e, 0x0a, 0x11, 0x41, 0x74, 0x6f, 0x6d, - 0x69, 0x63, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x22, 0x2e, - 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x74, 0x6f, 0x6d, 0x69, 0x63, 0x52, - 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x74, 0x6f, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x31, 0x0a, 0x15, 0x69, 0x66, 0x5f, 0x6e, + 0x6f, 0x74, 0x5f, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f, 0x61, 0x66, 0x74, 0x65, + 0x72, 0x18, 0x09, 0x20, 0x01, 0x28, 0x03, 0x52, 0x12, 0x69, 0x66, 0x4e, 0x6f, 0x74, 0x4d, 0x6f, + 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x41, 0x66, 0x74, 0x65, 0x72, 0x22, 0x2b, 0x0a, 0x13, 0x44, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xba, 0x01, 0x0a, 0x18, 0x41, 0x74, 0x6f, 0x6d, 0x69, 0x63, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x60, 0x0a, 0x11, 0x53, 0x74, 0x72, 0x65, - 0x61, 0x6d, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x22, 0x2e, - 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, - 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x72, - 0x65, 0x61, 0x6d, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x4f, 0x0a, 0x0c, 0x41, 0x73, - 0x73, 0x69, 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x1d, 0x2e, 0x66, 0x69, 0x6c, - 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4f, 0x0a, 0x0c, 0x4c, - 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x1d, 0x2e, 0x66, 0x69, - 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x66, 0x69, 0x6c, - 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x55, 0x0a, 0x0e, - 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1f, - 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x20, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0x5b, 0x0a, 0x10, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, - 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, - 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x66, 0x69, 0x6c, - 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x49, 0x0a, 0x0a, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x12, 0x1b, - 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, - 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x66, 0x69, - 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x37, 0x0a, 0x04, 0x50, - 0x69, 0x6e, 0x67, 0x12, 0x15, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x50, - 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x66, 0x69, 0x6c, - 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x6a, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x72, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x26, 0x2e, - 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, - 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, - 0x2e, 0x47, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x60, 0x0a, 0x11, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x22, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, - 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x69, 0x6c, 0x65, - 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x30, 0x01, 0x12, 0x65, 0x0a, 0x16, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x22, 0x2e, 0x66, - 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, - 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x23, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73, - 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x3a, 0x0a, 0x05, 0x4b, 0x76, 0x47, - 0x65, 0x74, 0x12, 0x16, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x76, - 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x66, 0x69, 0x6c, - 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x76, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3a, 0x0a, 0x05, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x12, 0x16, - 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, - 0x62, 0x2e, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x88, 0x01, 0x0a, 0x1f, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x6d, 0x6f, 0x74, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x6f, 0x6c, 0x64, 0x5f, 0x64, 0x69, 0x72, + 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6f, 0x6c, + 0x64, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x6f, 0x6c, + 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6f, 0x6c, + 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x6e, 0x65, 0x77, 0x5f, 0x64, 0x69, 0x72, + 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6e, 0x65, + 0x77, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x65, + 0x77, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x65, + 0x77, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x05, 0x52, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x73, 0x22, 0x1b, 0x0a, 0x19, 0x41, 0x74, 0x6f, 0x6d, 0x69, 0x63, 0x52, + 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0xba, 0x01, 0x0a, 0x18, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x6e, + 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x23, 0x0a, 0x0d, 0x6f, 0x6c, 0x64, 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6f, 0x6c, 0x64, 0x44, 0x69, 0x72, 0x65, 0x63, + 0x74, 0x6f, 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x6f, 0x6c, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6f, 0x6c, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, + 0x23, 0x0a, 0x0d, 0x6e, 0x65, 0x77, 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6e, 0x65, 0x77, 0x44, 0x69, 0x72, 0x65, 0x63, + 0x74, 0x6f, 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x65, 0x77, 0x5f, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x65, 0x77, 0x4e, 0x61, 0x6d, 0x65, 0x12, + 0x1e, 0x0a, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x05, 0x20, + 0x03, 0x28, 0x05, 0x52, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x22, + 0x9a, 0x01, 0x0a, 0x19, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1c, 0x0a, + 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x4a, 0x0a, 0x12, 0x65, + 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, + 0x70, 0x62, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x11, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x4e, 0x6f, 0x74, 0x69, 0x66, + 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x13, 0x0a, 0x05, 0x74, 0x73, 0x5f, 0x6e, 0x73, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x74, 0x73, 0x4e, 0x73, 0x22, 0x89, 0x02, 0x0a, + 0x13, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, + 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, + 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, + 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x17, 0x0a, 0x07, + 0x74, 0x74, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x74, + 0x74, 0x6c, 0x53, 0x65, 0x63, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x63, 0x65, + 0x6e, 0x74, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, + 0x43, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x61, + 0x63, 0x6b, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x61, 0x63, 0x6b, 0x12, 0x1b, + 0x0a, 0x09, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x08, 0x64, 0x61, 0x74, 0x61, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x64, + 0x69, 0x73, 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, + 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x22, 0xe1, 0x01, 0x0a, 0x14, 0x41, 0x73, 0x73, + 0x69, 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x17, 0x0a, 0x07, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, + 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, + 0x12, 0x12, 0x0a, 0x04, 0x61, 0x75, 0x74, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x61, 0x75, 0x74, 0x68, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, + 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x2e, 0x0a, 0x08, + 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x34, 0x0a, 0x13, + 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, + 0x64, 0x73, 0x22, 0x3d, 0x0a, 0x09, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, + 0x30, 0x0a, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x22, 0x79, 0x0a, 0x08, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, + 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, + 0x1d, 0x0a, 0x0a, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x55, 0x72, 0x6c, 0x12, 0x1b, + 0x0a, 0x09, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x08, 0x67, 0x72, 0x70, 0x63, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x64, + 0x61, 0x74, 0x61, 0x5f, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x22, 0xc3, 0x01, 0x0a, + 0x14, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x55, 0x0a, 0x0d, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x5f, 0x6d, 0x61, 0x70, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x66, + 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4c, 0x6f, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, + 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x4d, 0x61, 0x70, 0x1a, 0x54, 0x0a, 0x11, + 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x6b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x22, 0x20, 0x0a, 0x0a, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x7b, 0x0a, 0x15, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x34, 0x0a, + 0x16, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x6e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x5f, + 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x69, + 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x4e, 0x6f, 0x72, 0x6d, 0x61, 0x6c, 0x56, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x65, + 0x63, 0x5f, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x10, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x73, 0x22, 0x50, 0x0a, 0x16, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, + 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x0b, 0x63, + 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x14, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0b, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x22, 0x39, 0x0a, 0x17, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, + 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, + 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x1a, + 0x0a, 0x18, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x84, 0x01, 0x0a, 0x11, 0x53, + 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x74, 0x74, 0x6c, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x74, 0x79, 0x70, + 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x54, 0x79, 0x70, + 0x65, 0x22, 0x6f, 0x0a, 0x12, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x6f, 0x74, 0x61, 0x6c, + 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x74, 0x6f, 0x74, + 0x61, 0x6c, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x75, 0x73, 0x65, 0x64, 0x5f, 0x73, + 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x75, 0x73, 0x65, 0x64, 0x53, + 0x69, 0x7a, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, + 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x75, + 0x6e, 0x74, 0x22, 0x46, 0x0a, 0x0b, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, + 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, 0x7a, 0x0a, 0x0c, 0x50, 0x69, + 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x22, 0x0a, 0x0d, 0x73, 0x74, + 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x0b, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x4e, 0x73, 0x12, 0x24, + 0x0a, 0x0e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6e, 0x73, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x54, 0x69, + 0x6d, 0x65, 0x4e, 0x73, 0x12, 0x20, 0x0a, 0x0c, 0x73, 0x74, 0x6f, 0x70, 0x5f, 0x74, 0x69, 0x6d, + 0x65, 0x5f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x73, 0x74, 0x6f, 0x70, + 0x54, 0x69, 0x6d, 0x65, 0x4e, 0x73, 0x22, 0x1e, 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x46, 0x69, 0x6c, + 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xe8, 0x03, 0x0a, 0x1d, 0x47, 0x65, 0x74, 0x46, 0x69, + 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x61, 0x73, 0x74, + 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x61, 0x73, 0x74, 0x65, + 0x72, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x15, 0x0a, 0x06, 0x6d, 0x61, 0x78, 0x5f, 0x6d, 0x62, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6d, 0x61, 0x78, 0x4d, 0x62, 0x12, 0x1f, 0x0a, 0x0b, 0x64, + 0x69, 0x72, 0x5f, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0a, 0x64, 0x69, 0x72, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x16, 0x0a, 0x06, + 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x63, 0x69, + 0x70, 0x68, 0x65, 0x72, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x61, 0x64, + 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6d, 0x65, 0x74, + 0x72, 0x69, 0x63, 0x73, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x30, 0x0a, 0x14, 0x6d, + 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x5f, + 0x73, 0x65, 0x63, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, 0x52, 0x12, 0x6d, 0x65, 0x74, 0x72, 0x69, + 0x63, 0x73, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x53, 0x65, 0x63, 0x12, 0x18, 0x0a, + 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, + 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, + 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, + 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, + 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x66, 0x69, 0x6c, + 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x23, 0x0a, 0x0d, 0x6d, 0x61, 0x6a, 0x6f, 0x72, + 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, + 0x6d, 0x61, 0x6a, 0x6f, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x0a, 0x0d, + 0x6d, 0x69, 0x6e, 0x6f, 0x72, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0f, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x0c, 0x6d, 0x69, 0x6e, 0x6f, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x22, 0xb7, 0x02, 0x0a, 0x18, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, + 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, + 0x1f, 0x0a, 0x0b, 0x70, 0x61, 0x74, 0x68, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x70, 0x61, 0x74, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, + 0x12, 0x19, 0x0a, 0x08, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x07, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x4e, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x73, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, + 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x70, 0x61, 0x74, + 0x68, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x0c, 0x70, 0x61, 0x74, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x12, 0x1b, + 0x0a, 0x09, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x08, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x75, + 0x6e, 0x74, 0x69, 0x6c, 0x5f, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x75, + 0x6e, 0x74, 0x69, 0x6c, 0x4e, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x63, 0x6c, + 0x69, 0x65, 0x6e, 0x74, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x69, 0x72, + 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, + 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x22, 0x9a, 0x01, 0x0a, 0x19, + 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x72, + 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x69, + 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x4a, 0x0a, 0x12, 0x65, 0x76, 0x65, 0x6e, 0x74, + 0x5f, 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x45, + 0x76, 0x65, 0x6e, 0x74, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x52, 0x11, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x12, 0x13, 0x0a, 0x05, 0x74, 0x73, 0x5f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x04, 0x74, 0x73, 0x4e, 0x73, 0x22, 0x67, 0x0a, 0x1a, 0x54, 0x72, 0x61, 0x76, + 0x65, 0x72, 0x73, 0x65, 0x42, 0x66, 0x73, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, + 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, + 0x74, 0x6f, 0x72, 0x79, 0x12, 0x2b, 0x0a, 0x11, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x64, + 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x10, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x64, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, + 0x73, 0x22, 0x62, 0x0a, 0x1b, 0x54, 0x72, 0x61, 0x76, 0x65, 0x72, 0x73, 0x65, 0x42, 0x66, 0x73, + 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x25, + 0x0a, 0x05, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, + 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, + 0x65, 0x6e, 0x74, 0x72, 0x79, 0x22, 0x73, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x13, 0x0a, 0x05, 0x74, 0x73, 0x5f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x04, 0x74, 0x73, 0x4e, 0x73, 0x12, 0x2c, 0x0a, 0x12, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x10, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, + 0x48, 0x61, 0x73, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x65, 0x0a, 0x14, 0x4b, 0x65, + 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x70, + 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x67, 0x72, 0x70, 0x63, 0x50, + 0x6f, 0x72, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, + 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x73, 0x22, 0x17, 0x0a, 0x15, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x31, 0x0a, 0x13, 0x4c, 0x6f, + 0x63, 0x61, 0x74, 0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0xcd, 0x01, + 0x0a, 0x14, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x12, 0x45, 0x0a, 0x09, + 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x27, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, + 0x65, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, + 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x73, 0x1a, 0x58, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, + 0x25, 0x0a, 0x0e, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, + 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x67, 0x72, 0x70, 0x63, 0x41, 0x64, 0x64, + 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, + 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x20, 0x0a, + 0x0c, 0x4b, 0x76, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, + 0x3b, 0x0a, 0x0d, 0x4b, 0x76, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x36, 0x0a, 0x0c, + 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x22, 0x25, 0x0a, 0x0d, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xa4, 0x04, 0x0a, 0x09, + 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x3a, 0x0a, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, + 0x62, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x2e, 0x50, 0x61, 0x74, 0x68, + 0x43, 0x6f, 0x6e, 0x66, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, + 0xc0, 0x03, 0x0a, 0x08, 0x50, 0x61, 0x74, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x27, 0x0a, 0x0f, + 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, + 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x70, 0x6c, + 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x74, 0x6c, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, 0x74, 0x6c, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, + 0x6b, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, + 0x73, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x73, 0x79, 0x6e, 0x63, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x73, 0x79, 0x6e, 0x63, 0x12, 0x2e, 0x0a, 0x13, + 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x67, 0x72, 0x6f, 0x77, 0x74, 0x68, 0x5f, 0x63, 0x6f, + 0x75, 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x11, 0x76, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x47, 0x72, 0x6f, 0x77, 0x74, 0x68, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1b, 0x0a, 0x09, + 0x72, 0x65, 0x61, 0x64, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x08, 0x72, 0x65, 0x61, 0x64, 0x4f, 0x6e, 0x6c, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x61, 0x74, + 0x61, 0x5f, 0x63, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, + 0x64, 0x61, 0x74, 0x61, 0x43, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x61, + 0x63, 0x6b, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x61, 0x63, 0x6b, 0x12, 0x1b, + 0x0a, 0x09, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x08, 0x64, 0x61, 0x74, 0x61, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x2f, 0x0a, 0x14, 0x6d, + 0x61, 0x78, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x6c, 0x65, 0x6e, + 0x67, 0x74, 0x68, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x11, 0x6d, 0x61, 0x78, 0x46, 0x69, + 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x4c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x12, 0x34, 0x0a, 0x16, + 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x64, 0x65, + 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x64, 0x69, + 0x73, 0x61, 0x62, 0x6c, 0x65, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x69, + 0x6f, 0x6e, 0x22, 0x5a, 0x0a, 0x26, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x54, 0x6f, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x43, 0x6c, - 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x30, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, - 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x4f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x54, 0x6f, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, + 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, + 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x50, + 0x0a, 0x27, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x4f, 0x62, 0x6a, + 0x65, 0x63, 0x74, 0x54, 0x6f, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, + 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, 0x05, 0x65, 0x6e, 0x74, + 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, + 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x65, 0x6e, 0x74, 0x72, 0x79, + 0x22, 0x9b, 0x01, 0x0a, 0x0b, 0x4c, 0x6f, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x26, 0x0a, 0x0f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x5f, + 0x74, 0x6f, 0x5f, 0x6c, 0x6f, 0x63, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x73, + 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x54, 0x6f, 0x4c, 0x6f, 0x63, 0x6b, 0x12, 0x1f, 0x0a, 0x0b, + 0x72, 0x65, 0x6e, 0x65, 0x77, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0a, 0x72, 0x65, 0x6e, 0x65, 0x77, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x19, 0x0a, + 0x08, 0x69, 0x73, 0x5f, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x07, 0x69, 0x73, 0x4d, 0x6f, 0x76, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, + 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x22, 0x91, + 0x01, 0x0a, 0x0c, 0x4c, 0x6f, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x1f, 0x0a, 0x0b, 0x72, 0x65, 0x6e, 0x65, 0x77, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x6e, 0x65, 0x77, 0x54, 0x6f, 0x6b, 0x65, 0x6e, + 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x6b, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x12, + 0x2b, 0x0a, 0x12, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x68, 0x6f, 0x73, 0x74, 0x5f, 0x6d, 0x6f, 0x76, + 0x65, 0x64, 0x5f, 0x74, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x6c, 0x6f, 0x63, + 0x6b, 0x48, 0x6f, 0x73, 0x74, 0x4d, 0x6f, 0x76, 0x65, 0x64, 0x54, 0x6f, 0x12, 0x14, 0x0a, 0x05, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, + 0x6f, 0x72, 0x22, 0x5f, 0x0a, 0x0d, 0x55, 0x6e, 0x6c, 0x6f, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x72, 0x65, 0x6e, 0x65, 0x77, + 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, + 0x6e, 0x65, 0x77, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x19, 0x0a, 0x08, 0x69, 0x73, 0x5f, 0x6d, + 0x6f, 0x76, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x69, 0x73, 0x4d, 0x6f, + 0x76, 0x65, 0x64, 0x22, 0x41, 0x0a, 0x0e, 0x55, 0x6e, 0x6c, 0x6f, 0x63, 0x6b, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x19, 0x0a, 0x08, 0x6d, + 0x6f, 0x76, 0x65, 0x64, 0x5f, 0x74, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, + 0x6f, 0x76, 0x65, 0x64, 0x54, 0x6f, 0x22, 0x45, 0x0a, 0x14, 0x46, 0x69, 0x6e, 0x64, 0x4c, 0x6f, + 0x63, 0x6b, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, + 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x69, 0x73, 0x5f, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x69, 0x73, 0x4d, 0x6f, 0x76, 0x65, 0x64, 0x22, 0x2d, 0x0a, + 0x15, 0x46, 0x69, 0x6e, 0x64, 0x4c, 0x6f, 0x63, 0x6b, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x22, 0x75, 0x0a, 0x04, + 0x4c, 0x6f, 0x63, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x72, 0x65, 0x6e, 0x65, + 0x77, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, + 0x65, 0x6e, 0x65, 0x77, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x22, 0x0a, 0x0d, 0x65, 0x78, 0x70, + 0x69, 0x72, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x5f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x0b, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x64, 0x41, 0x74, 0x4e, 0x73, 0x12, 0x14, 0x0a, + 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6f, 0x77, + 0x6e, 0x65, 0x72, 0x22, 0x3c, 0x0a, 0x14, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x4c, + 0x6f, 0x63, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x05, 0x6c, + 0x6f, 0x63, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x66, 0x69, 0x6c, + 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x6b, 0x52, 0x05, 0x6c, 0x6f, 0x63, 0x6b, + 0x73, 0x22, 0x17, 0x0a, 0x15, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x4c, 0x6f, 0x63, + 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xf7, 0x10, 0x0a, 0x0c, 0x53, + 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x12, 0x67, 0x0a, 0x14, 0x4c, + 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x25, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, + 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x66, 0x69, 0x6c, + 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x44, 0x69, 0x72, 0x65, + 0x63, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x4e, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, + 0x69, 0x65, 0x73, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, + 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x30, 0x01, 0x12, 0x4c, 0x0a, 0x0b, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, + 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x4c, 0x0a, 0x0b, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x52, 0x0a, 0x0d, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x54, 0x6f, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x70, 0x70, + 0x65, 0x6e, 0x64, 0x54, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x70, 0x70, + 0x65, 0x6e, 0x64, 0x54, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x4c, 0x0a, 0x0b, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x5e, 0x0a, 0x11, 0x41, 0x74, 0x6f, 0x6d, 0x69, 0x63, 0x52, 0x65, 0x6e, 0x61, + 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x22, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, + 0x70, 0x62, 0x2e, 0x41, 0x74, 0x6f, 0x6d, 0x69, 0x63, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x69, + 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x74, 0x6f, 0x6d, 0x69, 0x63, 0x52, 0x65, 0x6e, + 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x60, 0x0a, 0x11, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x6e, 0x61, + 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x22, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, + 0x70, 0x62, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x69, + 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x6e, + 0x61, 0x6d, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x30, 0x01, 0x12, 0x4f, 0x0a, 0x0c, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x56, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, + 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, + 0x73, 0x73, 0x69, 0x67, 0x6e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4f, 0x0a, 0x0c, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x1d, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, + 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, + 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x55, 0x0a, 0x0e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, + 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, + 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x66, 0x69, 0x6c, 0x65, + 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, + 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5b, 0x0a, + 0x10, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x21, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, + 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x49, 0x0a, 0x0a, 0x53, 0x74, + 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x12, 0x1b, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, + 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, + 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x37, 0x0a, 0x04, 0x50, 0x69, 0x6e, 0x67, 0x12, 0x15, 0x2e, + 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, + 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6a, + 0x0a, 0x15, 0x47, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x26, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, + 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x27, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x69, + 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x66, 0x0a, 0x13, 0x54, 0x72, + 0x61, 0x76, 0x65, 0x72, 0x73, 0x65, 0x42, 0x66, 0x73, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x12, 0x24, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x72, 0x61, + 0x76, 0x65, 0x72, 0x73, 0x65, 0x42, 0x66, 0x73, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, + 0x70, 0x62, 0x2e, 0x54, 0x72, 0x61, 0x76, 0x65, 0x72, 0x73, 0x65, 0x42, 0x66, 0x73, 0x4d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x30, 0x01, 0x12, 0x60, 0x0a, 0x11, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x22, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, + 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x69, + 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, + 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x30, 0x01, 0x12, 0x65, 0x0a, 0x16, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, + 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x22, + 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, + 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x75, + 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x3a, 0x0a, 0x05, 0x4b, + 0x76, 0x47, 0x65, 0x74, 0x12, 0x16, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, + 0x4b, 0x76, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x66, + 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x76, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3a, 0x0a, 0x05, 0x4b, 0x76, 0x50, 0x75, 0x74, + 0x12, 0x16, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x76, 0x50, 0x75, + 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, + 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x76, 0x50, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x12, 0x88, 0x01, 0x0a, 0x1f, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x6d, + 0x6f, 0x74, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x54, 0x6f, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x30, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x54, 0x6f, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x43, 0x6c, 0x75, 0x73, 0x74, - 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x0f, - 0x44, 0x69, 0x73, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x64, 0x4c, 0x6f, 0x63, 0x6b, 0x12, - 0x15, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x6b, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, - 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x48, 0x0a, 0x11, 0x44, 0x69, 0x73, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x64, 0x55, - 0x6e, 0x6c, 0x6f, 0x63, 0x6b, 0x12, 0x17, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, - 0x2e, 0x55, 0x6e, 0x6c, 0x6f, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, - 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x55, 0x6e, 0x6c, 0x6f, 0x63, 0x6b, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x52, 0x0a, 0x0d, 0x46, 0x69, - 0x6e, 0x64, 0x4c, 0x6f, 0x63, 0x6b, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x1e, 0x2e, 0x66, 0x69, - 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x46, 0x69, 0x6e, 0x64, 0x4c, 0x6f, 0x63, 0x6b, 0x4f, - 0x77, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x66, 0x69, - 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x46, 0x69, 0x6e, 0x64, 0x4c, 0x6f, 0x63, 0x6b, 0x4f, - 0x77, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x52, - 0x0a, 0x0d, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x4c, 0x6f, 0x63, 0x6b, 0x73, 0x12, - 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, - 0x66, 0x65, 0x72, 0x4c, 0x6f, 0x63, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, - 0x66, 0x65, 0x72, 0x4c, 0x6f, 0x63, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x42, 0x4f, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, - 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x50, 0x72, 0x6f, - 0x74, 0x6f, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, - 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, - 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x66, 0x69, 0x6c, 0x65, 0x72, - 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x66, 0x69, 0x6c, 0x65, + 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, + 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x54, 0x6f, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x43, 0x6c, 0x75, + 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x42, + 0x0a, 0x0f, 0x44, 0x69, 0x73, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x64, 0x4c, 0x6f, 0x63, + 0x6b, 0x12, 0x15, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, + 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, + 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x48, 0x0a, 0x11, 0x44, 0x69, 0x73, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, + 0x64, 0x55, 0x6e, 0x6c, 0x6f, 0x63, 0x6b, 0x12, 0x17, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, + 0x70, 0x62, 0x2e, 0x55, 0x6e, 0x6c, 0x6f, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x18, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x55, 0x6e, 0x6c, 0x6f, + 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x52, 0x0a, 0x0d, + 0x46, 0x69, 0x6e, 0x64, 0x4c, 0x6f, 0x63, 0x6b, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x1e, 0x2e, + 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x46, 0x69, 0x6e, 0x64, 0x4c, 0x6f, 0x63, + 0x6b, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, + 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x46, 0x69, 0x6e, 0x64, 0x4c, 0x6f, 0x63, + 0x6b, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x52, 0x0a, 0x0d, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x4c, 0x6f, 0x63, 0x6b, + 0x73, 0x12, 0x1e, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x72, 0x61, + 0x6e, 0x73, 0x66, 0x65, 0x72, 0x4c, 0x6f, 0x63, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1f, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x72, 0x61, + 0x6e, 0x73, 0x66, 0x65, 0x72, 0x4c, 0x6f, 0x63, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x42, 0x4f, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, + 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x72, 0x50, + 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, + 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x66, 0x69, 0x6c, + 0x65, 0x72, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -5044,7 +5205,7 @@ func file_filer_proto_rawDescGZIP() []byte { return file_filer_proto_rawDescData } -var file_filer_proto_msgTypes = make([]protoimpl.MessageInfo, 68) +var file_filer_proto_msgTypes = make([]protoimpl.MessageInfo, 70) var file_filer_proto_goTypes = []interface{}{ (*LookupDirectoryEntryRequest)(nil), // 0: filer_pb.LookupDirectoryEntryRequest (*LookupDirectoryEntryResponse)(nil), // 1: filer_pb.LookupDirectoryEntryResponse @@ -5089,38 +5250,40 @@ var file_filer_proto_goTypes = []interface{}{ (*GetFilerConfigurationResponse)(nil), // 40: filer_pb.GetFilerConfigurationResponse (*SubscribeMetadataRequest)(nil), // 41: filer_pb.SubscribeMetadataRequest (*SubscribeMetadataResponse)(nil), // 42: filer_pb.SubscribeMetadataResponse - (*LogEntry)(nil), // 43: filer_pb.LogEntry - (*KeepConnectedRequest)(nil), // 44: filer_pb.KeepConnectedRequest - (*KeepConnectedResponse)(nil), // 45: filer_pb.KeepConnectedResponse - (*LocateBrokerRequest)(nil), // 46: filer_pb.LocateBrokerRequest - (*LocateBrokerResponse)(nil), // 47: filer_pb.LocateBrokerResponse - (*KvGetRequest)(nil), // 48: filer_pb.KvGetRequest - (*KvGetResponse)(nil), // 49: filer_pb.KvGetResponse - (*KvPutRequest)(nil), // 50: filer_pb.KvPutRequest - (*KvPutResponse)(nil), // 51: filer_pb.KvPutResponse - (*FilerConf)(nil), // 52: filer_pb.FilerConf - (*CacheRemoteObjectToLocalClusterRequest)(nil), // 53: filer_pb.CacheRemoteObjectToLocalClusterRequest - (*CacheRemoteObjectToLocalClusterResponse)(nil), // 54: filer_pb.CacheRemoteObjectToLocalClusterResponse - (*LockRequest)(nil), // 55: filer_pb.LockRequest - (*LockResponse)(nil), // 56: filer_pb.LockResponse - (*UnlockRequest)(nil), // 57: filer_pb.UnlockRequest - (*UnlockResponse)(nil), // 58: filer_pb.UnlockResponse - (*FindLockOwnerRequest)(nil), // 59: filer_pb.FindLockOwnerRequest - (*FindLockOwnerResponse)(nil), // 60: filer_pb.FindLockOwnerResponse - (*Lock)(nil), // 61: filer_pb.Lock - (*TransferLocksRequest)(nil), // 62: filer_pb.TransferLocksRequest - (*TransferLocksResponse)(nil), // 63: filer_pb.TransferLocksResponse - nil, // 64: filer_pb.Entry.ExtendedEntry - nil, // 65: filer_pb.LookupVolumeResponse.LocationsMapEntry - (*LocateBrokerResponse_Resource)(nil), // 66: filer_pb.LocateBrokerResponse.Resource - (*FilerConf_PathConf)(nil), // 67: filer_pb.FilerConf.PathConf + (*TraverseBfsMetadataRequest)(nil), // 43: filer_pb.TraverseBfsMetadataRequest + (*TraverseBfsMetadataResponse)(nil), // 44: filer_pb.TraverseBfsMetadataResponse + (*LogEntry)(nil), // 45: filer_pb.LogEntry + (*KeepConnectedRequest)(nil), // 46: filer_pb.KeepConnectedRequest + (*KeepConnectedResponse)(nil), // 47: filer_pb.KeepConnectedResponse + (*LocateBrokerRequest)(nil), // 48: filer_pb.LocateBrokerRequest + (*LocateBrokerResponse)(nil), // 49: filer_pb.LocateBrokerResponse + (*KvGetRequest)(nil), // 50: filer_pb.KvGetRequest + (*KvGetResponse)(nil), // 51: filer_pb.KvGetResponse + (*KvPutRequest)(nil), // 52: filer_pb.KvPutRequest + (*KvPutResponse)(nil), // 53: filer_pb.KvPutResponse + (*FilerConf)(nil), // 54: filer_pb.FilerConf + (*CacheRemoteObjectToLocalClusterRequest)(nil), // 55: filer_pb.CacheRemoteObjectToLocalClusterRequest + (*CacheRemoteObjectToLocalClusterResponse)(nil), // 56: filer_pb.CacheRemoteObjectToLocalClusterResponse + (*LockRequest)(nil), // 57: filer_pb.LockRequest + (*LockResponse)(nil), // 58: filer_pb.LockResponse + (*UnlockRequest)(nil), // 59: filer_pb.UnlockRequest + (*UnlockResponse)(nil), // 60: filer_pb.UnlockResponse + (*FindLockOwnerRequest)(nil), // 61: filer_pb.FindLockOwnerRequest + (*FindLockOwnerResponse)(nil), // 62: filer_pb.FindLockOwnerResponse + (*Lock)(nil), // 63: filer_pb.Lock + (*TransferLocksRequest)(nil), // 64: filer_pb.TransferLocksRequest + (*TransferLocksResponse)(nil), // 65: filer_pb.TransferLocksResponse + nil, // 66: filer_pb.Entry.ExtendedEntry + nil, // 67: filer_pb.LookupVolumeResponse.LocationsMapEntry + (*LocateBrokerResponse_Resource)(nil), // 68: filer_pb.LocateBrokerResponse.Resource + (*FilerConf_PathConf)(nil), // 69: filer_pb.FilerConf.PathConf } var file_filer_proto_depIdxs = []int32{ 5, // 0: filer_pb.LookupDirectoryEntryResponse.entry:type_name -> filer_pb.Entry 5, // 1: filer_pb.ListEntriesResponse.entry:type_name -> filer_pb.Entry 8, // 2: filer_pb.Entry.chunks:type_name -> filer_pb.FileChunk 11, // 3: filer_pb.Entry.attributes:type_name -> filer_pb.FuseAttributes - 64, // 4: filer_pb.Entry.extended:type_name -> filer_pb.Entry.ExtendedEntry + 66, // 4: filer_pb.Entry.extended:type_name -> filer_pb.Entry.ExtendedEntry 4, // 5: filer_pb.Entry.remote_entry:type_name -> filer_pb.RemoteEntry 5, // 6: filer_pb.FullEntry.entry:type_name -> filer_pb.Entry 5, // 7: filer_pb.EventNotification.old_entry:type_name -> filer_pb.Entry @@ -5134,67 +5297,70 @@ var file_filer_proto_depIdxs = []int32{ 7, // 15: filer_pb.StreamRenameEntryResponse.event_notification:type_name -> filer_pb.EventNotification 28, // 16: filer_pb.AssignVolumeResponse.location:type_name -> filer_pb.Location 28, // 17: filer_pb.Locations.locations:type_name -> filer_pb.Location - 65, // 18: filer_pb.LookupVolumeResponse.locations_map:type_name -> filer_pb.LookupVolumeResponse.LocationsMapEntry + 67, // 18: filer_pb.LookupVolumeResponse.locations_map:type_name -> filer_pb.LookupVolumeResponse.LocationsMapEntry 30, // 19: filer_pb.CollectionListResponse.collections:type_name -> filer_pb.Collection 7, // 20: filer_pb.SubscribeMetadataResponse.event_notification:type_name -> filer_pb.EventNotification - 66, // 21: filer_pb.LocateBrokerResponse.resources:type_name -> filer_pb.LocateBrokerResponse.Resource - 67, // 22: filer_pb.FilerConf.locations:type_name -> filer_pb.FilerConf.PathConf - 5, // 23: filer_pb.CacheRemoteObjectToLocalClusterResponse.entry:type_name -> filer_pb.Entry - 61, // 24: filer_pb.TransferLocksRequest.locks:type_name -> filer_pb.Lock - 27, // 25: filer_pb.LookupVolumeResponse.LocationsMapEntry.value:type_name -> filer_pb.Locations - 0, // 26: filer_pb.SeaweedFiler.LookupDirectoryEntry:input_type -> filer_pb.LookupDirectoryEntryRequest - 2, // 27: filer_pb.SeaweedFiler.ListEntries:input_type -> filer_pb.ListEntriesRequest - 12, // 28: filer_pb.SeaweedFiler.CreateEntry:input_type -> filer_pb.CreateEntryRequest - 14, // 29: filer_pb.SeaweedFiler.UpdateEntry:input_type -> filer_pb.UpdateEntryRequest - 16, // 30: filer_pb.SeaweedFiler.AppendToEntry:input_type -> filer_pb.AppendToEntryRequest - 18, // 31: filer_pb.SeaweedFiler.DeleteEntry:input_type -> filer_pb.DeleteEntryRequest - 20, // 32: filer_pb.SeaweedFiler.AtomicRenameEntry:input_type -> filer_pb.AtomicRenameEntryRequest - 22, // 33: filer_pb.SeaweedFiler.StreamRenameEntry:input_type -> filer_pb.StreamRenameEntryRequest - 24, // 34: filer_pb.SeaweedFiler.AssignVolume:input_type -> filer_pb.AssignVolumeRequest - 26, // 35: filer_pb.SeaweedFiler.LookupVolume:input_type -> filer_pb.LookupVolumeRequest - 31, // 36: filer_pb.SeaweedFiler.CollectionList:input_type -> filer_pb.CollectionListRequest - 33, // 37: filer_pb.SeaweedFiler.DeleteCollection:input_type -> filer_pb.DeleteCollectionRequest - 35, // 38: filer_pb.SeaweedFiler.Statistics:input_type -> filer_pb.StatisticsRequest - 37, // 39: filer_pb.SeaweedFiler.Ping:input_type -> filer_pb.PingRequest - 39, // 40: filer_pb.SeaweedFiler.GetFilerConfiguration:input_type -> filer_pb.GetFilerConfigurationRequest - 41, // 41: filer_pb.SeaweedFiler.SubscribeMetadata:input_type -> filer_pb.SubscribeMetadataRequest - 41, // 42: filer_pb.SeaweedFiler.SubscribeLocalMetadata:input_type -> filer_pb.SubscribeMetadataRequest - 48, // 43: filer_pb.SeaweedFiler.KvGet:input_type -> filer_pb.KvGetRequest - 50, // 44: filer_pb.SeaweedFiler.KvPut:input_type -> filer_pb.KvPutRequest - 53, // 45: filer_pb.SeaweedFiler.CacheRemoteObjectToLocalCluster:input_type -> filer_pb.CacheRemoteObjectToLocalClusterRequest - 55, // 46: filer_pb.SeaweedFiler.DistributedLock:input_type -> filer_pb.LockRequest - 57, // 47: filer_pb.SeaweedFiler.DistributedUnlock:input_type -> filer_pb.UnlockRequest - 59, // 48: filer_pb.SeaweedFiler.FindLockOwner:input_type -> filer_pb.FindLockOwnerRequest - 62, // 49: filer_pb.SeaweedFiler.TransferLocks:input_type -> filer_pb.TransferLocksRequest - 1, // 50: filer_pb.SeaweedFiler.LookupDirectoryEntry:output_type -> filer_pb.LookupDirectoryEntryResponse - 3, // 51: filer_pb.SeaweedFiler.ListEntries:output_type -> filer_pb.ListEntriesResponse - 13, // 52: filer_pb.SeaweedFiler.CreateEntry:output_type -> filer_pb.CreateEntryResponse - 15, // 53: filer_pb.SeaweedFiler.UpdateEntry:output_type -> filer_pb.UpdateEntryResponse - 17, // 54: filer_pb.SeaweedFiler.AppendToEntry:output_type -> filer_pb.AppendToEntryResponse - 19, // 55: filer_pb.SeaweedFiler.DeleteEntry:output_type -> filer_pb.DeleteEntryResponse - 21, // 56: filer_pb.SeaweedFiler.AtomicRenameEntry:output_type -> filer_pb.AtomicRenameEntryResponse - 23, // 57: filer_pb.SeaweedFiler.StreamRenameEntry:output_type -> filer_pb.StreamRenameEntryResponse - 25, // 58: filer_pb.SeaweedFiler.AssignVolume:output_type -> filer_pb.AssignVolumeResponse - 29, // 59: filer_pb.SeaweedFiler.LookupVolume:output_type -> filer_pb.LookupVolumeResponse - 32, // 60: filer_pb.SeaweedFiler.CollectionList:output_type -> filer_pb.CollectionListResponse - 34, // 61: filer_pb.SeaweedFiler.DeleteCollection:output_type -> filer_pb.DeleteCollectionResponse - 36, // 62: filer_pb.SeaweedFiler.Statistics:output_type -> filer_pb.StatisticsResponse - 38, // 63: filer_pb.SeaweedFiler.Ping:output_type -> filer_pb.PingResponse - 40, // 64: filer_pb.SeaweedFiler.GetFilerConfiguration:output_type -> filer_pb.GetFilerConfigurationResponse - 42, // 65: filer_pb.SeaweedFiler.SubscribeMetadata:output_type -> filer_pb.SubscribeMetadataResponse - 42, // 66: filer_pb.SeaweedFiler.SubscribeLocalMetadata:output_type -> filer_pb.SubscribeMetadataResponse - 49, // 67: filer_pb.SeaweedFiler.KvGet:output_type -> filer_pb.KvGetResponse - 51, // 68: filer_pb.SeaweedFiler.KvPut:output_type -> filer_pb.KvPutResponse - 54, // 69: filer_pb.SeaweedFiler.CacheRemoteObjectToLocalCluster:output_type -> filer_pb.CacheRemoteObjectToLocalClusterResponse - 56, // 70: filer_pb.SeaweedFiler.DistributedLock:output_type -> filer_pb.LockResponse - 58, // 71: filer_pb.SeaweedFiler.DistributedUnlock:output_type -> filer_pb.UnlockResponse - 60, // 72: filer_pb.SeaweedFiler.FindLockOwner:output_type -> filer_pb.FindLockOwnerResponse - 63, // 73: filer_pb.SeaweedFiler.TransferLocks:output_type -> filer_pb.TransferLocksResponse - 50, // [50:74] is the sub-list for method output_type - 26, // [26:50] is the sub-list for method input_type - 26, // [26:26] is the sub-list for extension type_name - 26, // [26:26] is the sub-list for extension extendee - 0, // [0:26] is the sub-list for field type_name + 5, // 21: filer_pb.TraverseBfsMetadataResponse.entry:type_name -> filer_pb.Entry + 68, // 22: filer_pb.LocateBrokerResponse.resources:type_name -> filer_pb.LocateBrokerResponse.Resource + 69, // 23: filer_pb.FilerConf.locations:type_name -> filer_pb.FilerConf.PathConf + 5, // 24: filer_pb.CacheRemoteObjectToLocalClusterResponse.entry:type_name -> filer_pb.Entry + 63, // 25: filer_pb.TransferLocksRequest.locks:type_name -> filer_pb.Lock + 27, // 26: filer_pb.LookupVolumeResponse.LocationsMapEntry.value:type_name -> filer_pb.Locations + 0, // 27: filer_pb.SeaweedFiler.LookupDirectoryEntry:input_type -> filer_pb.LookupDirectoryEntryRequest + 2, // 28: filer_pb.SeaweedFiler.ListEntries:input_type -> filer_pb.ListEntriesRequest + 12, // 29: filer_pb.SeaweedFiler.CreateEntry:input_type -> filer_pb.CreateEntryRequest + 14, // 30: filer_pb.SeaweedFiler.UpdateEntry:input_type -> filer_pb.UpdateEntryRequest + 16, // 31: filer_pb.SeaweedFiler.AppendToEntry:input_type -> filer_pb.AppendToEntryRequest + 18, // 32: filer_pb.SeaweedFiler.DeleteEntry:input_type -> filer_pb.DeleteEntryRequest + 20, // 33: filer_pb.SeaweedFiler.AtomicRenameEntry:input_type -> filer_pb.AtomicRenameEntryRequest + 22, // 34: filer_pb.SeaweedFiler.StreamRenameEntry:input_type -> filer_pb.StreamRenameEntryRequest + 24, // 35: filer_pb.SeaweedFiler.AssignVolume:input_type -> filer_pb.AssignVolumeRequest + 26, // 36: filer_pb.SeaweedFiler.LookupVolume:input_type -> filer_pb.LookupVolumeRequest + 31, // 37: filer_pb.SeaweedFiler.CollectionList:input_type -> filer_pb.CollectionListRequest + 33, // 38: filer_pb.SeaweedFiler.DeleteCollection:input_type -> filer_pb.DeleteCollectionRequest + 35, // 39: filer_pb.SeaweedFiler.Statistics:input_type -> filer_pb.StatisticsRequest + 37, // 40: filer_pb.SeaweedFiler.Ping:input_type -> filer_pb.PingRequest + 39, // 41: filer_pb.SeaweedFiler.GetFilerConfiguration:input_type -> filer_pb.GetFilerConfigurationRequest + 43, // 42: filer_pb.SeaweedFiler.TraverseBfsMetadata:input_type -> filer_pb.TraverseBfsMetadataRequest + 41, // 43: filer_pb.SeaweedFiler.SubscribeMetadata:input_type -> filer_pb.SubscribeMetadataRequest + 41, // 44: filer_pb.SeaweedFiler.SubscribeLocalMetadata:input_type -> filer_pb.SubscribeMetadataRequest + 50, // 45: filer_pb.SeaweedFiler.KvGet:input_type -> filer_pb.KvGetRequest + 52, // 46: filer_pb.SeaweedFiler.KvPut:input_type -> filer_pb.KvPutRequest + 55, // 47: filer_pb.SeaweedFiler.CacheRemoteObjectToLocalCluster:input_type -> filer_pb.CacheRemoteObjectToLocalClusterRequest + 57, // 48: filer_pb.SeaweedFiler.DistributedLock:input_type -> filer_pb.LockRequest + 59, // 49: filer_pb.SeaweedFiler.DistributedUnlock:input_type -> filer_pb.UnlockRequest + 61, // 50: filer_pb.SeaweedFiler.FindLockOwner:input_type -> filer_pb.FindLockOwnerRequest + 64, // 51: filer_pb.SeaweedFiler.TransferLocks:input_type -> filer_pb.TransferLocksRequest + 1, // 52: filer_pb.SeaweedFiler.LookupDirectoryEntry:output_type -> filer_pb.LookupDirectoryEntryResponse + 3, // 53: filer_pb.SeaweedFiler.ListEntries:output_type -> filer_pb.ListEntriesResponse + 13, // 54: filer_pb.SeaweedFiler.CreateEntry:output_type -> filer_pb.CreateEntryResponse + 15, // 55: filer_pb.SeaweedFiler.UpdateEntry:output_type -> filer_pb.UpdateEntryResponse + 17, // 56: filer_pb.SeaweedFiler.AppendToEntry:output_type -> filer_pb.AppendToEntryResponse + 19, // 57: filer_pb.SeaweedFiler.DeleteEntry:output_type -> filer_pb.DeleteEntryResponse + 21, // 58: filer_pb.SeaweedFiler.AtomicRenameEntry:output_type -> filer_pb.AtomicRenameEntryResponse + 23, // 59: filer_pb.SeaweedFiler.StreamRenameEntry:output_type -> filer_pb.StreamRenameEntryResponse + 25, // 60: filer_pb.SeaweedFiler.AssignVolume:output_type -> filer_pb.AssignVolumeResponse + 29, // 61: filer_pb.SeaweedFiler.LookupVolume:output_type -> filer_pb.LookupVolumeResponse + 32, // 62: filer_pb.SeaweedFiler.CollectionList:output_type -> filer_pb.CollectionListResponse + 34, // 63: filer_pb.SeaweedFiler.DeleteCollection:output_type -> filer_pb.DeleteCollectionResponse + 36, // 64: filer_pb.SeaweedFiler.Statistics:output_type -> filer_pb.StatisticsResponse + 38, // 65: filer_pb.SeaweedFiler.Ping:output_type -> filer_pb.PingResponse + 40, // 66: filer_pb.SeaweedFiler.GetFilerConfiguration:output_type -> filer_pb.GetFilerConfigurationResponse + 44, // 67: filer_pb.SeaweedFiler.TraverseBfsMetadata:output_type -> filer_pb.TraverseBfsMetadataResponse + 42, // 68: filer_pb.SeaweedFiler.SubscribeMetadata:output_type -> filer_pb.SubscribeMetadataResponse + 42, // 69: filer_pb.SeaweedFiler.SubscribeLocalMetadata:output_type -> filer_pb.SubscribeMetadataResponse + 51, // 70: filer_pb.SeaweedFiler.KvGet:output_type -> filer_pb.KvGetResponse + 53, // 71: filer_pb.SeaweedFiler.KvPut:output_type -> filer_pb.KvPutResponse + 56, // 72: filer_pb.SeaweedFiler.CacheRemoteObjectToLocalCluster:output_type -> filer_pb.CacheRemoteObjectToLocalClusterResponse + 58, // 73: filer_pb.SeaweedFiler.DistributedLock:output_type -> filer_pb.LockResponse + 60, // 74: filer_pb.SeaweedFiler.DistributedUnlock:output_type -> filer_pb.UnlockResponse + 62, // 75: filer_pb.SeaweedFiler.FindLockOwner:output_type -> filer_pb.FindLockOwnerResponse + 65, // 76: filer_pb.SeaweedFiler.TransferLocks:output_type -> filer_pb.TransferLocksResponse + 52, // [52:77] is the sub-list for method output_type + 27, // [27:52] is the sub-list for method input_type + 27, // [27:27] is the sub-list for extension type_name + 27, // [27:27] is the sub-list for extension extendee + 0, // [0:27] is the sub-list for field type_name } func init() { file_filer_proto_init() } @@ -5720,7 +5886,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LogEntry); i { + switch v := v.(*TraverseBfsMetadataRequest); i { case 0: return &v.state case 1: @@ -5732,7 +5898,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[44].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*KeepConnectedRequest); i { + switch v := v.(*TraverseBfsMetadataResponse); i { case 0: return &v.state case 1: @@ -5744,7 +5910,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[45].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*KeepConnectedResponse); i { + switch v := v.(*LogEntry); i { case 0: return &v.state case 1: @@ -5756,7 +5922,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[46].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LocateBrokerRequest); i { + switch v := v.(*KeepConnectedRequest); i { case 0: return &v.state case 1: @@ -5768,7 +5934,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[47].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LocateBrokerResponse); i { + switch v := v.(*KeepConnectedResponse); i { case 0: return &v.state case 1: @@ -5780,7 +5946,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[48].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*KvGetRequest); i { + switch v := v.(*LocateBrokerRequest); i { case 0: return &v.state case 1: @@ -5792,7 +5958,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*KvGetResponse); i { + switch v := v.(*LocateBrokerResponse); i { case 0: return &v.state case 1: @@ -5804,7 +5970,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*KvPutRequest); i { + switch v := v.(*KvGetRequest); i { case 0: return &v.state case 1: @@ -5816,7 +5982,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[51].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*KvPutResponse); i { + switch v := v.(*KvGetResponse); i { case 0: return &v.state case 1: @@ -5828,7 +5994,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[52].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FilerConf); i { + switch v := v.(*KvPutRequest); i { case 0: return &v.state case 1: @@ -5840,7 +6006,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[53].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CacheRemoteObjectToLocalClusterRequest); i { + switch v := v.(*KvPutResponse); i { case 0: return &v.state case 1: @@ -5852,7 +6018,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[54].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CacheRemoteObjectToLocalClusterResponse); i { + switch v := v.(*FilerConf); i { case 0: return &v.state case 1: @@ -5864,7 +6030,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[55].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LockRequest); i { + switch v := v.(*CacheRemoteObjectToLocalClusterRequest); i { case 0: return &v.state case 1: @@ -5876,7 +6042,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[56].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LockResponse); i { + switch v := v.(*CacheRemoteObjectToLocalClusterResponse); i { case 0: return &v.state case 1: @@ -5888,7 +6054,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[57].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*UnlockRequest); i { + switch v := v.(*LockRequest); i { case 0: return &v.state case 1: @@ -5900,7 +6066,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[58].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*UnlockResponse); i { + switch v := v.(*LockResponse); i { case 0: return &v.state case 1: @@ -5912,7 +6078,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[59].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FindLockOwnerRequest); i { + switch v := v.(*UnlockRequest); i { case 0: return &v.state case 1: @@ -5924,7 +6090,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[60].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FindLockOwnerResponse); i { + switch v := v.(*UnlockResponse); i { case 0: return &v.state case 1: @@ -5936,7 +6102,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[61].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Lock); i { + switch v := v.(*FindLockOwnerRequest); i { case 0: return &v.state case 1: @@ -5948,7 +6114,7 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[62].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TransferLocksRequest); i { + switch v := v.(*FindLockOwnerResponse); i { case 0: return &v.state case 1: @@ -5960,6 +6126,30 @@ func file_filer_proto_init() { } } file_filer_proto_msgTypes[63].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Lock); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_filer_proto_msgTypes[64].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TransferLocksRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_filer_proto_msgTypes[65].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*TransferLocksResponse); i { case 0: return &v.state @@ -5971,7 +6161,7 @@ func file_filer_proto_init() { return nil } } - file_filer_proto_msgTypes[66].Exporter = func(v interface{}, i int) interface{} { + file_filer_proto_msgTypes[68].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*LocateBrokerResponse_Resource); i { case 0: return &v.state @@ -5983,7 +6173,7 @@ func file_filer_proto_init() { return nil } } - file_filer_proto_msgTypes[67].Exporter = func(v interface{}, i int) interface{} { + file_filer_proto_msgTypes[69].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*FilerConf_PathConf); i { case 0: return &v.state @@ -6002,7 +6192,7 @@ func file_filer_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_filer_proto_rawDesc, NumEnums: 0, - NumMessages: 68, + NumMessages: 70, NumExtensions: 0, NumServices: 1, }, diff --git a/weed/pb/filer_pb/filer_client_bfs.go b/weed/pb/filer_pb/filer_client_bfs.go index cb9367a7d..046920940 100644 --- a/weed/pb/filer_pb/filer_client_bfs.go +++ b/weed/pb/filer_pb/filer_client_bfs.go @@ -1,7 +1,10 @@ package filer_pb import ( + "context" "fmt" + "github.com/seaweedfs/seaweedfs/weed/glog" + "io" "sync" "time" @@ -12,7 +15,7 @@ func TraverseBfs(filerClient FilerClient, parentPath util.FullPath, fn func(pare K := 5 var jobQueueWg sync.WaitGroup - queue := util.NewQueue() + queue := util.NewQueue[util.FullPath]() jobQueueWg.Add(1) queue.Enqueue(parentPath) terminates := make([]chan bool, K) @@ -26,11 +29,11 @@ func TraverseBfs(filerClient FilerClient, parentPath util.FullPath, fn func(pare return default: t := queue.Dequeue() - if t == nil { + if t == "" { time.Sleep(329 * time.Millisecond) continue } - dir := t.(util.FullPath) + dir := t processErr := processOneDirectory(filerClient, dir, queue, &jobQueueWg, fn) if processErr != nil { err = processErr @@ -47,7 +50,7 @@ func TraverseBfs(filerClient FilerClient, parentPath util.FullPath, fn func(pare return } -func processOneDirectory(filerClient FilerClient, parentPath util.FullPath, queue *util.Queue, jobQueueWg *sync.WaitGroup, fn func(parentPath util.FullPath, entry *Entry)) (err error) { +func processOneDirectory(filerClient FilerClient, parentPath util.FullPath, queue *util.Queue[util.FullPath], jobQueueWg *sync.WaitGroup, fn func(parentPath util.FullPath, entry *Entry)) (err error) { return ReadDirAllEntries(filerClient, parentPath, "", func(entry *Entry, isLast bool) error { @@ -65,3 +68,28 @@ func processOneDirectory(filerClient FilerClient, parentPath util.FullPath, queu }) } + +func StreamBfs(client SeaweedFilerClient, dir util.FullPath, olderThanTsNs int64, fn func(parentPath util.FullPath, entry *Entry)error) (err error) { + glog.V(0).Infof("TraverseBfsMetadata %v if before %v", dir, time.Unix(0, olderThanTsNs)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + stream, err := client.TraverseBfsMetadata(ctx, &TraverseBfsMetadataRequest{ + Directory: string(dir), + }) + if err != nil { + return fmt.Errorf("traverse bfs metadata: %v", err) + } + for { + resp, err := stream.Recv() + if err != nil { + if err == io.EOF { + break + } + return fmt.Errorf("traverse bfs metadata: %v", err) + } + if err := fn(util.FullPath(resp.Directory), resp.Entry); err != nil { + return err + } + } + return nil +} diff --git a/weed/pb/filer_pb/filer_grpc.pb.go b/weed/pb/filer_pb/filer_grpc.pb.go index ae1564f43..87b8de33d 100644 --- a/weed/pb/filer_pb/filer_grpc.pb.go +++ b/weed/pb/filer_pb/filer_grpc.pb.go @@ -34,6 +34,7 @@ const ( SeaweedFiler_Statistics_FullMethodName = "/filer_pb.SeaweedFiler/Statistics" SeaweedFiler_Ping_FullMethodName = "/filer_pb.SeaweedFiler/Ping" SeaweedFiler_GetFilerConfiguration_FullMethodName = "/filer_pb.SeaweedFiler/GetFilerConfiguration" + SeaweedFiler_TraverseBfsMetadata_FullMethodName = "/filer_pb.SeaweedFiler/TraverseBfsMetadata" SeaweedFiler_SubscribeMetadata_FullMethodName = "/filer_pb.SeaweedFiler/SubscribeMetadata" SeaweedFiler_SubscribeLocalMetadata_FullMethodName = "/filer_pb.SeaweedFiler/SubscribeLocalMetadata" SeaweedFiler_KvGet_FullMethodName = "/filer_pb.SeaweedFiler/KvGet" @@ -64,6 +65,7 @@ type SeaweedFilerClient interface { Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error) Ping(ctx context.Context, in *PingRequest, opts ...grpc.CallOption) (*PingResponse, error) GetFilerConfiguration(ctx context.Context, in *GetFilerConfigurationRequest, opts ...grpc.CallOption) (*GetFilerConfigurationResponse, error) + TraverseBfsMetadata(ctx context.Context, in *TraverseBfsMetadataRequest, opts ...grpc.CallOption) (SeaweedFiler_TraverseBfsMetadataClient, error) SubscribeMetadata(ctx context.Context, in *SubscribeMetadataRequest, opts ...grpc.CallOption) (SeaweedFiler_SubscribeMetadataClient, error) SubscribeLocalMetadata(ctx context.Context, in *SubscribeMetadataRequest, opts ...grpc.CallOption) (SeaweedFiler_SubscribeLocalMetadataClient, error) KvGet(ctx context.Context, in *KvGetRequest, opts ...grpc.CallOption) (*KvGetResponse, error) @@ -265,8 +267,40 @@ func (c *seaweedFilerClient) GetFilerConfiguration(ctx context.Context, in *GetF return out, nil } +func (c *seaweedFilerClient) TraverseBfsMetadata(ctx context.Context, in *TraverseBfsMetadataRequest, opts ...grpc.CallOption) (SeaweedFiler_TraverseBfsMetadataClient, error) { + stream, err := c.cc.NewStream(ctx, &SeaweedFiler_ServiceDesc.Streams[2], SeaweedFiler_TraverseBfsMetadata_FullMethodName, opts...) + if err != nil { + return nil, err + } + x := &seaweedFilerTraverseBfsMetadataClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type SeaweedFiler_TraverseBfsMetadataClient interface { + Recv() (*TraverseBfsMetadataResponse, error) + grpc.ClientStream +} + +type seaweedFilerTraverseBfsMetadataClient struct { + grpc.ClientStream +} + +func (x *seaweedFilerTraverseBfsMetadataClient) Recv() (*TraverseBfsMetadataResponse, error) { + m := new(TraverseBfsMetadataResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + func (c *seaweedFilerClient) SubscribeMetadata(ctx context.Context, in *SubscribeMetadataRequest, opts ...grpc.CallOption) (SeaweedFiler_SubscribeMetadataClient, error) { - stream, err := c.cc.NewStream(ctx, &SeaweedFiler_ServiceDesc.Streams[2], SeaweedFiler_SubscribeMetadata_FullMethodName, opts...) + stream, err := c.cc.NewStream(ctx, &SeaweedFiler_ServiceDesc.Streams[3], SeaweedFiler_SubscribeMetadata_FullMethodName, opts...) if err != nil { return nil, err } @@ -298,7 +332,7 @@ func (x *seaweedFilerSubscribeMetadataClient) Recv() (*SubscribeMetadataResponse } func (c *seaweedFilerClient) SubscribeLocalMetadata(ctx context.Context, in *SubscribeMetadataRequest, opts ...grpc.CallOption) (SeaweedFiler_SubscribeLocalMetadataClient, error) { - stream, err := c.cc.NewStream(ctx, &SeaweedFiler_ServiceDesc.Streams[3], SeaweedFiler_SubscribeLocalMetadata_FullMethodName, opts...) + stream, err := c.cc.NewStream(ctx, &SeaweedFiler_ServiceDesc.Streams[4], SeaweedFiler_SubscribeLocalMetadata_FullMethodName, opts...) if err != nil { return nil, err } @@ -411,6 +445,7 @@ type SeaweedFilerServer interface { Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error) Ping(context.Context, *PingRequest) (*PingResponse, error) GetFilerConfiguration(context.Context, *GetFilerConfigurationRequest) (*GetFilerConfigurationResponse, error) + TraverseBfsMetadata(*TraverseBfsMetadataRequest, SeaweedFiler_TraverseBfsMetadataServer) error SubscribeMetadata(*SubscribeMetadataRequest, SeaweedFiler_SubscribeMetadataServer) error SubscribeLocalMetadata(*SubscribeMetadataRequest, SeaweedFiler_SubscribeLocalMetadataServer) error KvGet(context.Context, *KvGetRequest) (*KvGetResponse, error) @@ -473,6 +508,9 @@ func (UnimplementedSeaweedFilerServer) Ping(context.Context, *PingRequest) (*Pin func (UnimplementedSeaweedFilerServer) GetFilerConfiguration(context.Context, *GetFilerConfigurationRequest) (*GetFilerConfigurationResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetFilerConfiguration not implemented") } +func (UnimplementedSeaweedFilerServer) TraverseBfsMetadata(*TraverseBfsMetadataRequest, SeaweedFiler_TraverseBfsMetadataServer) error { + return status.Errorf(codes.Unimplemented, "method TraverseBfsMetadata not implemented") +} func (UnimplementedSeaweedFilerServer) SubscribeMetadata(*SubscribeMetadataRequest, SeaweedFiler_SubscribeMetadataServer) error { return status.Errorf(codes.Unimplemented, "method SubscribeMetadata not implemented") } @@ -789,6 +827,27 @@ func _SeaweedFiler_GetFilerConfiguration_Handler(srv interface{}, ctx context.Co return interceptor(ctx, in, info, handler) } +func _SeaweedFiler_TraverseBfsMetadata_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(TraverseBfsMetadataRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(SeaweedFilerServer).TraverseBfsMetadata(m, &seaweedFilerTraverseBfsMetadataServer{stream}) +} + +type SeaweedFiler_TraverseBfsMetadataServer interface { + Send(*TraverseBfsMetadataResponse) error + grpc.ServerStream +} + +type seaweedFilerTraverseBfsMetadataServer struct { + grpc.ServerStream +} + +func (x *seaweedFilerTraverseBfsMetadataServer) Send(m *TraverseBfsMetadataResponse) error { + return x.ServerStream.SendMsg(m) +} + func _SeaweedFiler_SubscribeMetadata_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(SubscribeMetadataRequest) if err := stream.RecvMsg(m); err != nil { @@ -1057,6 +1116,11 @@ var SeaweedFiler_ServiceDesc = grpc.ServiceDesc{ ServerStreams: true, }, { + StreamName: "TraverseBfsMetadata", + Handler: _SeaweedFiler_TraverseBfsMetadata_Handler, + ServerStreams: true, + }, + { StreamName: "SubscribeMetadata", Handler: _SeaweedFiler_SubscribeMetadata_Handler, ServerStreams: true, diff --git a/weed/pb/filer_pb_tail.go b/weed/pb/filer_pb_tail.go index b54dad871..b7cca7585 100644 --- a/weed/pb/filer_pb_tail.go +++ b/weed/pb/filer_pb_tail.go @@ -17,6 +17,7 @@ const ( TrivialOnError EventErrorType = iota FatalOnError RetryForeverOnError + DontLogError ) // MetadataFollowOption is used to control the behavior of the metadata following @@ -96,6 +97,8 @@ func makeSubscribeMetadataFunc(option *MetadataFollowOption, processEventFn Proc glog.Errorf("process %v: %v", resp, err) return true }) + case DontLogError: + // pass default: glog.Errorf("process %v: %v", resp, err) } diff --git a/weed/remote_storage/traverse_bfs.go b/weed/remote_storage/traverse_bfs.go deleted file mode 100644 index 983555f6c..000000000 --- a/weed/remote_storage/traverse_bfs.go +++ /dev/null @@ -1,62 +0,0 @@ -package remote_storage - -import ( - "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" - "github.com/seaweedfs/seaweedfs/weed/util" - "sync" - "time" -) - -type ListDirectoryFunc func(parentDir util.FullPath, visitFn VisitFunc) error - -func TraverseBfs(listDirFn ListDirectoryFunc, parentPath util.FullPath, visitFn VisitFunc) (err error) { - K := 5 - - var dirQueueWg sync.WaitGroup - dirQueue := util.NewQueue() - dirQueueWg.Add(1) - dirQueue.Enqueue(parentPath) - var isTerminating bool - - for i := 0; i < K; i++ { - go func() { - for { - if isTerminating { - break - } - t := dirQueue.Dequeue() - if t == nil { - time.Sleep(329 * time.Millisecond) - continue - } - dir := t.(util.FullPath) - processErr := processOneDirectory(listDirFn, dir, visitFn, dirQueue, &dirQueueWg) - if processErr != nil { - err = processErr - } - dirQueueWg.Done() - } - }() - } - - dirQueueWg.Wait() - isTerminating = true - return - -} - -func processOneDirectory(listDirFn ListDirectoryFunc, parentPath util.FullPath, visitFn VisitFunc, dirQueue *util.Queue, dirQueueWg *sync.WaitGroup) error { - - return listDirFn(parentPath, func(dir string, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error { - if err := visitFn(dir, name, isDirectory, remoteEntry); err != nil { - return err - } - if !isDirectory { - return nil - } - dirQueueWg.Add(1) - dirQueue.Enqueue(parentPath.Child(name)) - return nil - }) - -} diff --git a/weed/replication/repl_util/replication_util.go b/weed/replication/repl_util/replication_util.go index 9682ca623..4a77fd04a 100644 --- a/weed/replication/repl_util/replication_util.go +++ b/weed/replication/repl_util/replication_util.go @@ -4,7 +4,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/replication/source" - "github.com/seaweedfs/seaweedfs/weed/util" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) func CopyFromChunkViews(chunkViews *filer.IntervalList[*filer.ChunkView], filerSource *source.FilerSource, writeFunc func(data []byte) error) error { @@ -21,7 +21,7 @@ func CopyFromChunkViews(chunkViews *filer.IntervalList[*filer.ChunkView], filerS var shouldRetry bool for _, fileUrl := range fileUrls { - shouldRetry, err = util.ReadUrlAsStream(fileUrl, chunk.CipherKey, chunk.IsGzipped, chunk.IsFullChunk(), chunk.OffsetInChunk, int(chunk.ViewSize), func(data []byte) { + shouldRetry, err = util_http.ReadUrlAsStream(fileUrl, chunk.CipherKey, chunk.IsGzipped, chunk.IsFullChunk(), chunk.OffsetInChunk, int(chunk.ViewSize), func(data []byte) { writeErr = writeFunc(data) }) if err != nil { diff --git a/weed/replication/sink/azuresink/azure_sink.go b/weed/replication/sink/azuresink/azure_sink.go index 9bbd7b8eb..890e68fd4 100644 --- a/weed/replication/sink/azuresink/azure_sink.go +++ b/weed/replication/sink/azuresink/azure_sink.go @@ -5,8 +5,10 @@ import ( "context" "fmt" "github.com/seaweedfs/seaweedfs/weed/replication/repl_util" + "net/http" "net/url" "strings" + "time" "github.com/Azure/azure-storage-blob-go/azblob" "github.com/seaweedfs/seaweedfs/weed/filer" @@ -109,7 +111,16 @@ func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry, signatures [] // Azure Storage account's container. appendBlobURL := g.containerURL.NewAppendBlobURL(key) - _, err := appendBlobURL.Create(context.Background(), azblob.BlobHTTPHeaders{}, azblob.Metadata{}, azblob.BlobAccessConditions{}, azblob.BlobTagsMap{}, azblob.ClientProvidedKeyOptions{}, azblob.ImmutabilityPolicyOptions{}) + accessCondition := azblob.BlobAccessConditions{} + if entry.Attributes!=nil && entry.Attributes.Mtime>0 { + accessCondition.ModifiedAccessConditions.IfUnmodifiedSince = time.Unix(entry.Attributes.Mtime, 0) + } + + res, err := appendBlobURL.Create(context.Background(), azblob.BlobHTTPHeaders{}, azblob.Metadata{}, accessCondition, azblob.BlobTagsMap{}, azblob.ClientProvidedKeyOptions{}, azblob.ImmutabilityPolicyOptions{}) + if res != nil && res.StatusCode() == http.StatusPreconditionFailed { + glog.V(0).Infof("skip overwriting %s/%s: %v", g.container, key, err) + return nil + } if err != nil { return err } diff --git a/weed/replication/sink/filersink/fetch_write.go b/weed/replication/sink/filersink/fetch_write.go index 63e1226b6..4bcbc7898 100644 --- a/weed/replication/sink/filersink/fetch_write.go +++ b/weed/replication/sink/filersink/fetch_write.go @@ -14,6 +14,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) func (fs *FilerSink) replicateChunks(sourceChunks []*filer_pb.FileChunk, path string) (replicatedChunks []*filer_pb.FileChunk, err error) { @@ -88,9 +89,15 @@ func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk, path string) if err != nil { return "", fmt.Errorf("read part %s: %v", sourceChunk.GetFileIdString(), err) } - defer util.CloseResponse(resp) + defer util_http.CloseResponse(resp) - fileId, uploadResult, err, _ := operation.UploadWithRetry( + uploader, err := operation.NewUploader() + if err != nil { + glog.V(0).Infof("upload source data %v: %v", sourceChunk.GetFileIdString(), err) + return "", fmt.Errorf("upload data: %v", err) + } + + fileId, uploadResult, err, _ := uploader.UploadWithRetry( fs, &filer_pb.AssignVolumeRequest{ Count: 1, diff --git a/weed/replication/sink/filersink/filer_sink.go b/weed/replication/sink/filersink/filer_sink.go index ce2de41b9..49f6877a0 100644 --- a/weed/replication/sink/filersink/filer_sink.go +++ b/weed/replication/sink/filersink/filer_sink.go @@ -120,6 +120,10 @@ func (fs *FilerSink) CreateEntry(key string, entry *filer_pb.Entry, signatures [ glog.V(3).Infof("already replicated %s", key) return nil } + if resp.Entry.Attributes != nil && resp.Entry.Attributes.Mtime >= entry.Attributes.Mtime { + glog.V(3).Infof("skip overwriting %s", key) + return nil + } } replicatedChunks, err := fs.replicateChunks(entry.GetChunks(), key) diff --git a/weed/replication/source/filer_source.go b/weed/replication/source/filer_source.go index 167907a5a..768e251a4 100644 --- a/weed/replication/source/filer_source.go +++ b/weed/replication/source/filer_source.go @@ -15,6 +15,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/util" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) type ReplicationSource interface { @@ -106,7 +107,7 @@ func (fs *FilerSource) LookupFileId(part string) (fileUrls []string, err error) func (fs *FilerSource) ReadPart(fileId string) (filename string, header http.Header, resp *http.Response, err error) { if fs.proxyByFiler { - return util.DownloadFile("http://"+fs.address+"/?proxyChunkId="+fileId, "") + return util_http.DownloadFile("http://"+fs.address+"/?proxyChunkId="+fileId, "") } fileUrls, err := fs.LookupFileId(fileId) @@ -115,7 +116,7 @@ func (fs *FilerSource) ReadPart(fileId string) (filename string, header http.Hea } for _, fileUrl := range fileUrls { - filename, header, resp, err = util.DownloadFile(fileUrl, "") + filename, header, resp, err = util_http.DownloadFile(fileUrl, "") if err != nil { glog.V(1).Infof("fail to read from %s: %v", fileUrl, err) } else { diff --git a/weed/s3api/AmazonS3.xsd b/weed/s3api/AmazonS3.xsd index 8016a6a83..8a0136b44 100644 --- a/weed/s3api/AmazonS3.xsd +++ b/weed/s3api/AmazonS3.xsd @@ -525,6 +525,7 @@ <xsd:element name="IsTruncated" type="xsd:boolean"/> <xsd:element name="Contents" type="tns:ListEntry" minOccurs="0" maxOccurs="unbounded"/> <xsd:element name="CommonPrefixes" type="tns:PrefixEntry" minOccurs="0" maxOccurs="unbounded"/> + <xsd:element name="EncodingType" type="xsd:string"/> </xsd:sequence> </xsd:complexType> diff --git a/weed/s3api/README.txt b/weed/s3api/README.txt index 10a18ff4d..f7eb1988a 100644 --- a/weed/s3api/README.txt +++ b/weed/s3api/README.txt @@ -1,7 +1,7 @@ see https://blog.aqwari.net/xml-schema-go/ 1. go get aqwari.net/xml/cmd/xsdgen -2. xsdgen -o s3api_xsd_generated.go -pkg s3api AmazonS3.xsd - - - +2. Add EncodingType element for ListBucketResult in AmazonS3.xsd +3. xsdgen -o s3api_xsd_generated.go -pkg s3api AmazonS3.xsd +4. Remove empty Grantee struct in s3api_xsd_generated.go +5. Remove xmlns: sed s'/http:\/\/s3.amazonaws.com\/doc\/2006-03-01\/\ //' s3api_xsd_generated.go diff --git a/weed/s3api/auto_signature_v4_test.go b/weed/s3api/auto_signature_v4_test.go index 6ff67b5bf..bb67c35c2 100644 --- a/weed/s3api/auto_signature_v4_test.go +++ b/weed/s3api/auto_signature_v4_test.go @@ -43,7 +43,7 @@ func TestIsRequestPresignedSignatureV4(t *testing.T) { for i, testCase := range testCases { // creating an input HTTP request. // Only the query parameters are relevant for this particular test. - inputReq, err := http.NewRequest("GET", "http://example.com", nil) + inputReq, err := http.NewRequest(http.MethodGet, "http://example.com", nil) if err != nil { t.Fatalf("Error initializing input HTTP request: %v", err) } @@ -85,9 +85,9 @@ func TestIsReqAuthenticated(t *testing.T) { s3Error s3err.ErrorCode }{ // When request is unsigned, access denied is returned. - {mustNewRequest("GET", "http://127.0.0.1:9000", 0, nil, t), s3err.ErrAccessDenied}, + {mustNewRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), s3err.ErrAccessDenied}, // When request is properly signed, error is none. - {mustNewSignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), s3err.ErrNone}, + {mustNewSignedRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), s3err.ErrNone}, } // Validates all testcases. @@ -117,8 +117,8 @@ func TestCheckaAnonymousRequestAuthType(t *testing.T) { ErrCode s3err.ErrorCode Action Action }{ - {Request: mustNewRequest("GET", "http://127.0.0.1:9000/bucket", 0, nil, t), ErrCode: s3err.ErrNone, Action: s3_constants.ACTION_READ}, - {Request: mustNewRequest("PUT", "http://127.0.0.1:9000/bucket", 0, nil, t), ErrCode: s3err.ErrAccessDenied, Action: s3_constants.ACTION_WRITE}, + {Request: mustNewRequest(http.MethodGet, "http://127.0.0.1:9000/bucket", 0, nil, t), ErrCode: s3err.ErrNone, Action: s3_constants.ACTION_READ}, + {Request: mustNewRequest(http.MethodPut, "http://127.0.0.1:9000/bucket", 0, nil, t), ErrCode: s3err.ErrAccessDenied, Action: s3_constants.ACTION_WRITE}, } for i, testCase := range testCases { _, s3Error := iam.authRequest(testCase.Request, testCase.Action) @@ -155,9 +155,9 @@ func TestCheckAdminRequestAuthType(t *testing.T) { Request *http.Request ErrCode s3err.ErrorCode }{ - {Request: mustNewRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: s3err.ErrAccessDenied}, - {Request: mustNewSignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: s3err.ErrNone}, - {Request: mustNewPresignedRequest(iam, "GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: s3err.ErrNone}, + {Request: mustNewRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: s3err.ErrAccessDenied}, + {Request: mustNewSignedRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: s3err.ErrNone}, + {Request: mustNewPresignedRequest(iam, http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: s3err.ErrNone}, } for i, testCase := range testCases { if _, s3Error := iam.reqSignatureV4Verify(testCase.Request); s3Error != testCase.ErrCode { @@ -214,7 +214,7 @@ func mustNewPresignedRequest(iam *IdentityAccessManagement, method string, urlSt // Returns new HTTP request object. func newTestRequest(method, urlStr string, contentLength int64, body io.ReadSeeker) (*http.Request, error) { if method == "" { - method = "POST" + method = http.MethodPost } // Save for subsequent use diff --git a/weed/s3api/s3api_acl_helper.go b/weed/s3api/s3api_acl_helper.go index 0332b6a39..b9fb1131e 100644 --- a/weed/s3api/s3api_acl_helper.go +++ b/weed/s3api/s3api_acl_helper.go @@ -9,9 +9,9 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" - "github.com/seaweedfs/seaweedfs/weed/util" "net/http" "strings" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) type AccountManager interface { @@ -32,7 +32,7 @@ func GetAccountId(r *http.Request) string { // ExtractAcl extracts the acl from the request body, or from the header if request body is empty func ExtractAcl(r *http.Request, accountManager AccountManager, ownership, bucketOwnerId, ownerId, accountId string) (grants []*s3.Grant, errCode s3err.ErrorCode) { if r.Body != nil && r.Body != http.NoBody { - defer util.CloseRequest(r) + defer util_http.CloseRequest(r) var acp s3.AccessControlPolicy err := xmlutil.UnmarshalXML(&acp, xml.NewDecoder(r.Body), "") diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index 12d2c0432..7d0d76ea4 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -13,7 +13,6 @@ import ( "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil" "github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket" - "github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" @@ -26,14 +25,9 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) -type ListAllMyBucketsResult struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult"` - Owner *s3.Owner - Buckets []*s3.Bucket `xml:"Buckets>Bucket"` -} - func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { glog.V(3).Infof("ListBucketsHandler") @@ -59,25 +53,25 @@ func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Reques identityId := r.Header.Get(s3_constants.AmzIdentityId) - var buckets []*s3.Bucket + var listBuckets ListAllMyBucketsList for _, entry := range entries { if entry.IsDirectory { if identity != nil && !identity.canDo(s3_constants.ACTION_LIST, entry.Name, "") { continue } - buckets = append(buckets, &s3.Bucket{ - Name: aws.String(entry.Name), - CreationDate: aws.Time(time.Unix(entry.Attributes.Crtime, 0).UTC()), + listBuckets.Bucket = append(listBuckets.Bucket, ListAllMyBucketsEntry{ + Name: entry.Name, + CreationDate: time.Unix(entry.Attributes.Crtime, 0).UTC(), }) } } response = ListAllMyBucketsResult{ - Owner: &s3.Owner{ - ID: aws.String(identityId), - DisplayName: aws.String(identityId), + Owner: CanonicalUser{ + ID: identityId, + DisplayName: identityId, }, - Buckets: buckets, + Buckets: listBuckets, } writeSuccessResponseXML(w, r, response) @@ -461,7 +455,11 @@ func (s3a *S3ApiServer) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *h for prefix, ttl := range collectionTtls { bucketPrefix := fmt.Sprintf("%s/%s/", s3a.option.BucketsPath, bucket) if strings.HasPrefix(prefix, bucketPrefix) && strings.HasSuffix(ttl, "d") { - fc.DeleteLocationConf(prefix) + pathConf, found := fc.GetLocationConf(prefix) + if found { + pathConf.Ttl = "" + fc.SetLocationConf(pathConf) + } changed = true } } @@ -487,7 +485,7 @@ func (s3a *S3ApiServer) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *h // GetBucketLocationHandler Get bucket location // https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLocation.html func (s3a *S3ApiServer) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) { - writeSuccessResponseXML(w, r, LocationConstraint{}) + writeSuccessResponseXML(w, r, CreateBucketConfiguration{}) } // GetBucketRequestPaymentHandler Get bucket location @@ -513,7 +511,7 @@ func (s3a *S3ApiServer) PutBucketOwnershipControls(w http.ResponseWriter, r *htt } var v s3.OwnershipControls - defer util.CloseRequest(r) + defer util_http.CloseRequest(r) err := xmlutil.UnmarshalXML(&v, xml.NewDecoder(r.Body), "") if err != nil { diff --git a/weed/s3api/s3api_bucket_handlers_test.go b/weed/s3api/s3api_bucket_handlers_test.go index 1cff45aa0..2c8a3ae2c 100644 --- a/weed/s3api/s3api_bucket_handlers_test.go +++ b/weed/s3api/s3api_bucket_handlers_test.go @@ -4,37 +4,34 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "testing" "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/s3" ) func TestListBucketsHandler(t *testing.T) { expected := `<?xml version="1.0" encoding="UTF-8"?> -<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><DisplayName></DisplayName><ID></ID></Owner><Buckets><Bucket><CreationDate>2011-04-09T12:34:49Z</CreationDate><Name>test1</Name></Bucket><Bucket><CreationDate>2011-02-09T12:34:49Z</CreationDate><Name>test2</Name></Bucket></Buckets></ListAllMyBucketsResult>` +<ListAllMyBucketsResult><Owner><ID></ID></Owner><Buckets><Bucket><Name>test1</Name><CreationDate>2011-04-09T12:34:49Z</CreationDate></Bucket><Bucket><Name>test2</Name><CreationDate>2011-02-09T12:34:49Z</CreationDate></Bucket></Buckets></ListAllMyBucketsResult>` var response ListAllMyBucketsResult - var buckets []*s3.Bucket - buckets = append(buckets, &s3.Bucket{ - Name: aws.String("test1"), - CreationDate: aws.Time(time.Date(2011, 4, 9, 12, 34, 49, 0, time.UTC)), + var bucketsList ListAllMyBucketsList + bucketsList.Bucket = append(bucketsList.Bucket, ListAllMyBucketsEntry{ + Name: "test1", + CreationDate: time.Date(2011, 4, 9, 12, 34, 49, 0, time.UTC), }) - buckets = append(buckets, &s3.Bucket{ - Name: aws.String("test2"), - CreationDate: aws.Time(time.Date(2011, 2, 9, 12, 34, 49, 0, time.UTC)), + bucketsList.Bucket = append(bucketsList.Bucket, ListAllMyBucketsEntry{ + Name: "test2", + CreationDate: time.Date(2011, 2, 9, 12, 34, 49, 0, time.UTC), }) response = ListAllMyBucketsResult{ - Owner: &s3.Owner{ - ID: aws.String(""), - DisplayName: aws.String(""), + Owner: CanonicalUser{ + ID: "", + DisplayName: "", }, - Buckets: buckets, + Buckets: bucketsList, } encoded := string(s3err.EncodeXMLResponse(response)) if encoded != expected { - t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected) + t.Errorf("unexpected output:%s\nexpecting:%s", encoded, expected) } } diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 1d58af8bc..3ab72285f 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -3,6 +3,8 @@ package s3api import ( "bytes" "fmt" + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "io" "net/http" "net/url" @@ -14,7 +16,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/util/mem" "github.com/seaweedfs/seaweedfs/weed/glog" - "github.com/seaweedfs/seaweedfs/weed/util" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) func mimeDetect(r *http.Request, dataReader io.Reader) io.ReadCloser { @@ -35,10 +37,17 @@ func urlEscapeObject(object string) string { return "/" + t } +func entryUrlEncode(dir string, entry string, encodingTypeUrl bool) (dirName string, entryName string, prefix string) { + if !encodingTypeUrl { + return dir, entry, entry + } + return urlPathEscape(dir), url.QueryEscape(entry), urlPathEscape(entry) +} + func urlPathEscape(object string) string { var escapedParts []string for _, part := range strings.Split(object, "/") { - escapedParts = append(escapedParts, url.PathEscape(part)) + escapedParts = append(escapedParts, strings.ReplaceAll(url.PathEscape(part), "+", "%2B")) } return strings.Join(escapedParts, "/") } @@ -63,6 +72,37 @@ func removeDuplicateSlashes(object string) string { return result.String() } +func newListEntry(entry *filer_pb.Entry, key string, dir string, name string, bucketPrefix string, fetchOwner bool, isDirectory bool, encodingTypeUrl bool) (listEntry ListEntry) { + storageClass := "STANDARD" + if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok { + storageClass = string(v) + } + keyFormat := "%s/%s" + if isDirectory { + keyFormat += "/" + } + if key == "" { + key = fmt.Sprintf(keyFormat, dir, name)[len(bucketPrefix):] + } + if encodingTypeUrl { + key = urlPathEscape(key) + } + listEntry = ListEntry{ + Key: key, + LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), + ETag: "\"" + filer.ETag(entry) + "\"", + Size: int64(filer.FileSize(entry)), + StorageClass: StorageClass(storageClass), + } + if fetchOwner { + listEntry.Owner = CanonicalUser{ + ID: fmt.Sprintf("%x", entry.Attributes.Uid), + DisplayName: entry.Attributes.UserName, + } + } + return listEntry +} + func (s3a *S3ApiServer) toFilerUrl(bucket, object string) string { object = urlPathEscape(removeDuplicateSlashes(object)) destUrl := fmt.Sprintf("http://%s%s/%s%s", @@ -131,7 +171,7 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return } - defer util.CloseResponse(resp) + defer util_http.CloseResponse(resp) if resp.StatusCode == http.StatusPreconditionFailed { s3err.WriteErrorResponse(w, r, s3err.ErrPreconditionFailed) @@ -143,7 +183,7 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des return } - if r.Method == "DELETE" { + if r.Method == http.MethodDelete { if resp.StatusCode == http.StatusNotFound { // this is normal responseStatusCode := responseFn(resp, w) diff --git a/weed/s3api/s3api_object_handlers_copy.go b/weed/s3api/s3api_object_handlers_copy.go index 8d13fe17e..4ca8010d2 100644 --- a/weed/s3api/s3api_object_handlers_copy.go +++ b/weed/s3api/s3api_object_handlers_copy.go @@ -14,6 +14,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "github.com/seaweedfs/seaweedfs/weed/util" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) const ( @@ -87,12 +88,12 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request srcUrl := fmt.Sprintf("http://%s%s/%s%s", s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, srcBucket, urlEscapeObject(srcObject)) - _, _, resp, err := util.DownloadFile(srcUrl, s3a.maybeGetFilerJwtAuthorizationToken(false)) + _, _, resp, err := util_http.DownloadFile(srcUrl, s3a.maybeGetFilerJwtAuthorizationToken(false)) if err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource) return } - defer util.CloseResponse(resp) + defer util_http.CloseResponse(resp) tagErr := processMetadata(r.Header, resp.Header, replaceMeta, replaceTagging, s3a.getTags, dir, name) if tagErr != nil { @@ -175,12 +176,12 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req srcUrl := fmt.Sprintf("http://%s%s/%s%s", s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, srcBucket, urlEscapeObject(srcObject)) - resp, dataReader, err := util.ReadUrlAsReaderCloser(srcUrl, s3a.maybeGetFilerJwtAuthorizationToken(false), rangeHeader) + resp, dataReader, err := util_http.ReadUrlAsReaderCloser(srcUrl, s3a.maybeGetFilerJwtAuthorizationToken(false), rangeHeader) if err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource) return } - defer util.CloseResponse(resp) + defer util_http.CloseResponse(resp) defer dataReader.Close() glog.V(2).Infof("copy from %s to %s", srcUrl, dstUrl) diff --git a/weed/s3api/s3api_object_handlers_delete.go b/weed/s3api/s3api_object_handlers_delete.go index 580578593..7656b9d38 100644 --- a/weed/s3api/s3api_object_handlers_delete.go +++ b/weed/s3api/s3api_object_handlers_delete.go @@ -27,14 +27,13 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque bucket, object := s3_constants.GetBucketAndObject(r) glog.V(3).Infof("DeleteObjectHandler %s %s", bucket, object) - object = urlPathEscape(removeDuplicateSlashes(object)) + target := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object)) + dir, name := target.DirAndName() - s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { - err := doDeleteEntry(client, s3a.option.BucketsPath+"/"+bucket, object, true, false) - if err != nil { - // skip deletion error, usually the file is not found - return nil + if err := doDeleteEntry(client, dir, name, true, false); err != nil { + return err } if s3a.option.AllowEmptyFolder { @@ -42,11 +41,8 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque } directoriesWithDeletion := make(map[string]int) - lastSeparator := strings.LastIndex(object, "/") - if lastSeparator > 0 { - parentDirectoryPath := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object[:lastSeparator]) - directoriesWithDeletion[parentDirectoryPath]++ - + if strings.LastIndex(object, "/") > 0 { + directoriesWithDeletion[dir]++ // purge empty folders, only checking folders with deletions for len(directoriesWithDeletion) > 0 { directoriesWithDeletion = s3a.doDeleteEmptyDirectories(client, directoriesWithDeletion) @@ -55,6 +51,10 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque return nil }) + if err != nil { + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + return + } w.WriteHeader(http.StatusNoContent) } diff --git a/weed/s3api/s3api_object_handlers_list.go b/weed/s3api/s3api_object_handlers_list.go index 38e7f6fef..27d18800c 100644 --- a/weed/s3api/s3api_object_handlers_list.go +++ b/weed/s3api/s3api_object_handlers_list.go @@ -4,33 +4,44 @@ import ( "context" "encoding/xml" "fmt" + "github.com/aws/aws-sdk-go/service/s3" "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "io" "net/http" "net/url" "strconv" "strings" - "time" - - "github.com/seaweedfs/seaweedfs/weed/filer" - "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" ) +type OptionalString struct { + string + set bool +} + +func (o OptionalString) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { + if !o.set { + return nil + } + return e.EncodeElement(o.string, startElement) +} + type ListBucketResultV2 struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"` - Name string `xml:"Name"` - Prefix string `xml:"Prefix"` - MaxKeys int `xml:"MaxKeys"` - Delimiter string `xml:"Delimiter,omitempty"` - IsTruncated bool `xml:"IsTruncated"` - Contents []ListEntry `xml:"Contents,omitempty"` - CommonPrefixes []PrefixEntry `xml:"CommonPrefixes,omitempty"` - ContinuationToken string `xml:"ContinuationToken,omitempty"` - NextContinuationToken string `xml:"NextContinuationToken,omitempty"` - KeyCount int `xml:"KeyCount"` - StartAfter string `xml:"StartAfter,omitempty"` + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix"` + MaxKeys uint16 `xml:"MaxKeys"` + Delimiter string `xml:"Delimiter,omitempty"` + IsTruncated bool `xml:"IsTruncated"` + Contents []ListEntry `xml:"Contents,omitempty"` + CommonPrefixes []PrefixEntry `xml:"CommonPrefixes,omitempty"` + ContinuationToken OptionalString `xml:"ContinuationToken,omitempty"` + NextContinuationToken string `xml:"NextContinuationToken,omitempty"` + EncodingType string `xml:"EncodingType,omitempty"` + KeyCount int `xml:"KeyCount"` + StartAfter string `xml:"StartAfter,omitempty"` } func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) { @@ -41,19 +52,19 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ bucket, _ := s3_constants.GetBucketAndObject(r) glog.V(3).Infof("ListObjectsV2Handler %s", bucket) - originalPrefix, continuationToken, startAfter, delimiter, _, maxKeys := getListObjectsV2Args(r.URL.Query()) + originalPrefix, startAfter, delimiter, continuationToken, encodingTypeUrl, fetchOwner, maxKeys := getListObjectsV2Args(r.URL.Query()) if maxKeys < 0 { s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys) return } - marker := continuationToken - if continuationToken == "" { + marker := continuationToken.string + if !continuationToken.set { marker = startAfter } - response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter) + response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter, encodingTypeUrl, fetchOwner) if err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) @@ -68,7 +79,6 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ } responseV2 := &ListBucketResultV2{ - XMLName: response.XMLName, Name: response.Name, CommonPrefixes: response.CommonPrefixes, Contents: response.Contents, @@ -76,11 +86,14 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ Delimiter: response.Delimiter, IsTruncated: response.IsTruncated, KeyCount: len(response.Contents) + len(response.CommonPrefixes), - MaxKeys: response.MaxKeys, + MaxKeys: uint16(response.MaxKeys), NextContinuationToken: response.NextMarker, Prefix: response.Prefix, StartAfter: startAfter, } + if encodingTypeUrl { + responseV2.EncodingType = s3.EncodingTypeUrl + } writeSuccessResponseXML(w, r, responseV2) } @@ -93,14 +106,13 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ bucket, _ := s3_constants.GetBucketAndObject(r) glog.V(3).Infof("ListObjectsV1Handler %s", bucket) - originalPrefix, marker, delimiter, maxKeys := getListObjectsV1Args(r.URL.Query()) + originalPrefix, marker, delimiter, encodingTypeUrl, maxKeys := getListObjectsV1Args(r.URL.Query()) if maxKeys < 0 { s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys) return } - - response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter) + response, err := s3a.listFilerEntries(bucket, originalPrefix, uint16(maxKeys), marker, delimiter, encodingTypeUrl, true) if err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) @@ -117,7 +129,7 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ writeSuccessResponseXML(w, r, response) } -func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, maxKeys int, originalMarker string, delimiter string) (response ListBucketResult, err error) { +func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, maxKeys uint16, originalMarker string, delimiter string, encodingTypeUrl bool, fetchOwner bool) (response ListBucketResult, err error) { // convert full path prefix into directory name and prefix for entry name requestDir, prefix, marker := normalizePrefixMarker(originalPrefix, originalMarker) bucketPrefix := fmt.Sprintf("%s/%s/", s3a.option.BucketsPath, bucket) @@ -141,23 +153,15 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m empty := true nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, cursor, marker, delimiter, false, func(dir string, entry *filer_pb.Entry) { empty = false + dirName, entryName, prefixName := entryUrlEncode(dir, entry.Name, encodingTypeUrl) if entry.IsDirectory { if entry.IsDirectoryKeyObject() { - contents = append(contents, ListEntry{ - Key: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):], - LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), - ETag: "\"" + filer.ETag(entry) + "\"", - Owner: CanonicalUser{ - ID: fmt.Sprintf("%x", entry.Attributes.Uid), - DisplayName: entry.Attributes.UserName, - }, - StorageClass: "STANDARD", - }) + contents = append(contents, newListEntry(entry, "", dirName, entryName, bucketPrefix, fetchOwner, true, false)) cursor.maxKeys-- // https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html } else if delimiter == "/" { // A response can contain CommonPrefixes only if you specify a delimiter. commonPrefixes = append(commonPrefixes, PrefixEntry{ - Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):], + Prefix: fmt.Sprintf("%s/%s/", dirName, prefixName)[len(bucketPrefix):], }) //All of the keys (up to 1,000) rolled up into a common prefix count as a single return when calculating the number of returns. cursor.maxKeys-- @@ -195,21 +199,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m } } if !delimiterFound { - storageClass := "STANDARD" - if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok { - storageClass = string(v) - } - contents = append(contents, ListEntry{ - Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):], - LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), - ETag: "\"" + filer.ETag(entry) + "\"", - Size: int64(filer.FileSize(entry)), - Owner: CanonicalUser{ - ID: fmt.Sprintf("%x", entry.Attributes.Uid), - DisplayName: entry.Attributes.UserName, - }, - StorageClass: StorageClass(storageClass), - }) + contents = append(contents, newListEntry(entry, "", dirName, entryName, bucketPrefix, fetchOwner, false, false)) cursor.maxKeys-- } } @@ -237,13 +227,17 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m Prefix: originalPrefix, Marker: originalMarker, NextMarker: nextMarker, - MaxKeys: maxKeys, + MaxKeys: int(maxKeys), Delimiter: delimiter, IsTruncated: cursor.isTruncated, Contents: contents, CommonPrefixes: commonPrefixes, } - + if encodingTypeUrl { + // Todo used for pass test_bucket_listv2_encoding_basic + // sort.Slice(response.CommonPrefixes, func(i, j int) bool { return response.CommonPrefixes[i].Prefix < response.CommonPrefixes[j].Prefix }) + response.EncodingType = s3.EncodingTypeUrl + } return nil }) @@ -251,7 +245,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m } type ListingCursor struct { - maxKeys int + maxKeys uint16 isTruncated bool prefixEndsOnDelimiter bool } @@ -434,13 +428,16 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d return } -func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int) { +func getListObjectsV2Args(values url.Values) (prefix, startAfter, delimiter string, token OptionalString, encodingTypeUrl bool, fetchOwner bool, maxkeys uint16) { prefix = values.Get("prefix") - token = values.Get("continuation-token") + token = OptionalString{set: values.Has("continuation-token"), string: values.Get("continuation-token")} startAfter = values.Get("start-after") delimiter = values.Get("delimiter") + encodingTypeUrl = values.Get("encoding-type") == s3.EncodingTypeUrl if values.Get("max-keys") != "" { - maxkeys, _ = strconv.Atoi(values.Get("max-keys")) + if maxKeys, err := strconv.ParseUint(values.Get("max-keys"), 10, 16); err == nil { + maxkeys = uint16(maxKeys) + } } else { maxkeys = maxObjectListSizeLimit } @@ -448,12 +445,15 @@ func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimit return } -func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int) { +func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, encodingTypeUrl bool, maxkeys int16) { prefix = values.Get("prefix") marker = values.Get("marker") delimiter = values.Get("delimiter") + encodingTypeUrl = values.Get("encoding-type") == "url" if values.Get("max-keys") != "" { - maxkeys, _ = strconv.Atoi(values.Get("max-keys")) + if maxKeys, err := strconv.ParseInt(values.Get("max-keys"), 10, 16); err == nil { + maxkeys = int16(maxKeys) + } } else { maxkeys = maxObjectListSizeLimit } diff --git a/weed/s3api/s3api_object_handlers_list_test.go b/weed/s3api/s3api_object_handlers_list_test.go index 6974c5567..3295c2fca 100644 --- a/weed/s3api/s3api_object_handlers_list_test.go +++ b/weed/s3api/s3api_object_handlers_list_test.go @@ -12,7 +12,7 @@ func TestListObjectsHandler(t *testing.T) { // https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html expected := `<?xml version="1.0" encoding="UTF-8"?> -<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>test_container</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Contents><Key>1.zip</Key><ETag>"4397da7a7649e8085de9916c240e8166"</ETag><Size>1234567</Size><Owner><ID>65a011niqo39cdf8ec533ec3d1ccaafsa932</ID></Owner><StorageClass>STANDARD</StorageClass><LastModified>2011-04-09T12:34:49Z</LastModified></Contents></ListBucketResult>` +<ListBucketResult><Name>test_container</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Contents><Key>1.zip</Key><ETag>"4397da7a7649e8085de9916c240e8166"</ETag><Size>1234567</Size><Owner><ID>65a011niqo39cdf8ec533ec3d1ccaafsa932</ID></Owner><StorageClass>STANDARD</StorageClass><LastModified>2011-04-09T12:34:49Z</LastModified></Contents><EncodingType></EncodingType></ListBucketResult>` response := ListBucketResult{ Name: "test_container", diff --git a/weed/s3api/s3api_object_handlers_put.go b/weed/s3api/s3api_object_handlers_put.go index 49d385afc..0f8743a30 100644 --- a/weed/s3api/s3api_object_handlers_put.go +++ b/weed/s3api/s3api_object_handlers_put.go @@ -110,7 +110,7 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader hash := md5.New() var body = io.TeeReader(dataReader, hash) - proxyReq, err := http.NewRequest("PUT", uploadUrl, body) + proxyReq, err := http.NewRequest(http.MethodPut, uploadUrl, body) if err != nil { glog.Errorf("NewRequest %s: %v", uploadUrl, err) diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 9422318ce..e0517ffb7 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -20,6 +20,8 @@ import ( "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/util" "google.golang.org/grpc" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" + util_http_client "github.com/seaweedfs/seaweedfs/weed/util/http/client" ) type S3ApiServerOption struct { @@ -44,7 +46,7 @@ type S3ApiServer struct { cb *CircuitBreaker randomClientId int32 filerGuard *security.Guard - client *http.Client + client util_http_client.HTTPClientInterface bucketRegistry *BucketRegistry } @@ -84,10 +86,9 @@ func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer } s3ApiServer.bucketRegistry = NewBucketRegistry(s3ApiServer) if option.LocalFilerSocket == "" { - s3ApiServer.client = &http.Client{Transport: &http.Transport{ - MaxIdleConns: 1024, - MaxIdleConnsPerHost: 1024, - }} + if s3ApiServer.client, err = util_http.NewGlobalHttpClient(); err != nil { + return nil, err + } } else { s3ApiServer.client = &http.Client{ Transport: &http.Transport{ @@ -109,9 +110,9 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { apiRouter := router.PathPrefix("/").Subrouter() // Readiness Probe - apiRouter.Methods("GET").Path("/status").HandlerFunc(s3a.StatusHandler) + apiRouter.Methods(http.MethodGet).Path("/status").HandlerFunc(s3a.StatusHandler) - apiRouter.Methods("OPTIONS").HandlerFunc( + apiRouter.Methods(http.MethodOptions).HandlerFunc( func(w http.ResponseWriter, r *http.Request) { origin := r.Header.Get("Origin") if origin != "" { @@ -161,135 +162,135 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { // objects with query // CopyObjectPart - bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", `.*?(\/|%2F).*?`).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.CopyObjectPartHandler, ACTION_WRITE)), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") + bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", `.*?(\/|%2F).*?`).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.CopyObjectPartHandler, ACTION_WRITE)), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") // PutObjectPart - bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectPartHandler, ACTION_WRITE)), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") + bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectPartHandler, ACTION_WRITE)), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") // CompleteMultipartUpload - bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.CompleteMultipartUploadHandler, ACTION_WRITE)), "POST")).Queries("uploadId", "{uploadId:.*}") + bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.CompleteMultipartUploadHandler, ACTION_WRITE)), "POST")).Queries("uploadId", "{uploadId:.*}") // NewMultipartUpload - bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.NewMultipartUploadHandler, ACTION_WRITE)), "POST")).Queries("uploads", "") + bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.NewMultipartUploadHandler, ACTION_WRITE)), "POST")).Queries("uploads", "") // AbortMultipartUpload - bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.AbortMultipartUploadHandler, ACTION_WRITE)), "DELETE")).Queries("uploadId", "{uploadId:.*}") + bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.AbortMultipartUploadHandler, ACTION_WRITE)), "DELETE")).Queries("uploadId", "{uploadId:.*}") // ListObjectParts - bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectPartsHandler, ACTION_READ)), "GET")).Queries("uploadId", "{uploadId:.*}") + bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectPartsHandler, ACTION_READ)), "GET")).Queries("uploadId", "{uploadId:.*}") // ListMultipartUploads - bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListMultipartUploadsHandler, ACTION_READ)), "GET")).Queries("uploads", "") + bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListMultipartUploadsHandler, ACTION_READ)), "GET")).Queries("uploads", "") // GetObjectTagging - bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetObjectTaggingHandler, ACTION_READ)), "GET")).Queries("tagging", "") + bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetObjectTaggingHandler, ACTION_READ)), "GET")).Queries("tagging", "") // PutObjectTagging - bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectTaggingHandler, ACTION_TAGGING)), "PUT")).Queries("tagging", "") + bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectTaggingHandler, ACTION_TAGGING)), "PUT")).Queries("tagging", "") // DeleteObjectTagging - bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteObjectTaggingHandler, ACTION_TAGGING)), "DELETE")).Queries("tagging", "") + bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteObjectTaggingHandler, ACTION_TAGGING)), "DELETE")).Queries("tagging", "") // PutObjectACL - bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectAclHandler, ACTION_WRITE_ACP)), "PUT")).Queries("acl", "") + bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectAclHandler, ACTION_WRITE_ACP)), "PUT")).Queries("acl", "") // PutObjectRetention - bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectRetentionHandler, ACTION_WRITE)), "PUT")).Queries("retention", "") + bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectRetentionHandler, ACTION_WRITE)), "PUT")).Queries("retention", "") // PutObjectLegalHold - bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectLegalHoldHandler, ACTION_WRITE)), "PUT")).Queries("legal-hold", "") + bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectLegalHoldHandler, ACTION_WRITE)), "PUT")).Queries("legal-hold", "") // PutObjectLockConfiguration - bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectLockConfigurationHandler, ACTION_WRITE)), "PUT")).Queries("object-lock", "") + bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectLockConfigurationHandler, ACTION_WRITE)), "PUT")).Queries("object-lock", "") // GetObjectACL - bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetObjectAclHandler, ACTION_READ_ACP)), "GET")).Queries("acl", "") + bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetObjectAclHandler, ACTION_READ_ACP)), "GET")).Queries("acl", "") // objects with query // raw objects // HeadObject - bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.HeadObjectHandler, ACTION_READ)), "GET")) + bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.HeadObjectHandler, ACTION_READ)), "GET")) // GetObject, but directory listing is not supported - bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetObjectHandler, ACTION_READ)), "GET")) + bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetObjectHandler, ACTION_READ)), "GET")) // CopyObject - bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.CopyObjectHandler, ACTION_WRITE)), "COPY")) + bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.CopyObjectHandler, ACTION_WRITE)), "COPY")) // PutObject - bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectHandler, ACTION_WRITE)), "PUT")) + bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectHandler, ACTION_WRITE)), "PUT")) // DeleteObject - bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteObjectHandler, ACTION_WRITE)), "DELETE")) + bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteObjectHandler, ACTION_WRITE)), "DELETE")) // raw objects // buckets with query // DeleteMultipleObjects - bucket.Methods("POST").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteMultipleObjectsHandler, ACTION_WRITE)), "DELETE")).Queries("delete", "") + bucket.Methods(http.MethodPost).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteMultipleObjectsHandler, ACTION_WRITE)), "DELETE")).Queries("delete", "") // GetBucketACL - bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketAclHandler, ACTION_READ_ACP)), "GET")).Queries("acl", "") + bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketAclHandler, ACTION_READ_ACP)), "GET")).Queries("acl", "") // PutBucketACL - bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketAclHandler, ACTION_WRITE_ACP)), "PUT")).Queries("acl", "") + bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketAclHandler, ACTION_WRITE_ACP)), "PUT")).Queries("acl", "") // GetBucketPolicy - bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketPolicyHandler, ACTION_READ)), "GET")).Queries("policy", "") + bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketPolicyHandler, ACTION_READ)), "GET")).Queries("policy", "") // PutBucketPolicy - bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketPolicyHandler, ACTION_WRITE)), "PUT")).Queries("policy", "") + bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketPolicyHandler, ACTION_WRITE)), "PUT")).Queries("policy", "") // DeleteBucketPolicy - bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketPolicyHandler, ACTION_WRITE)), "DELETE")).Queries("policy", "") + bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketPolicyHandler, ACTION_WRITE)), "DELETE")).Queries("policy", "") // GetBucketCors - bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketCorsHandler, ACTION_READ)), "GET")).Queries("cors", "") + bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketCorsHandler, ACTION_READ)), "GET")).Queries("cors", "") // PutBucketCors - bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketCorsHandler, ACTION_WRITE)), "PUT")).Queries("cors", "") + bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketCorsHandler, ACTION_WRITE)), "PUT")).Queries("cors", "") // DeleteBucketCors - bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketCorsHandler, ACTION_WRITE)), "DELETE")).Queries("cors", "") + bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketCorsHandler, ACTION_WRITE)), "DELETE")).Queries("cors", "") // GetBucketLifecycleConfiguration - bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketLifecycleConfigurationHandler, ACTION_READ)), "GET")).Queries("lifecycle", "") + bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketLifecycleConfigurationHandler, ACTION_READ)), "GET")).Queries("lifecycle", "") // PutBucketLifecycleConfiguration - bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketLifecycleConfigurationHandler, ACTION_WRITE)), "PUT")).Queries("lifecycle", "") + bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketLifecycleConfigurationHandler, ACTION_WRITE)), "PUT")).Queries("lifecycle", "") // DeleteBucketLifecycleConfiguration - bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketLifecycleHandler, ACTION_WRITE)), "DELETE")).Queries("lifecycle", "") + bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketLifecycleHandler, ACTION_WRITE)), "DELETE")).Queries("lifecycle", "") // GetBucketLocation - bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketLocationHandler, ACTION_READ)), "GET")).Queries("location", "") + bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketLocationHandler, ACTION_READ)), "GET")).Queries("location", "") // GetBucketRequestPayment - bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketRequestPaymentHandler, ACTION_READ)), "GET")).Queries("requestPayment", "") + bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketRequestPaymentHandler, ACTION_READ)), "GET")).Queries("requestPayment", "") // GetBucketVersioning - bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketVersioningHandler, ACTION_READ)), "GET")).Queries("versioning", "") - bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketVersioningHandler, ACTION_WRITE)), "PUT")).Queries("versioning", "") + bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketVersioningHandler, ACTION_READ)), "GET")).Queries("versioning", "") + bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketVersioningHandler, ACTION_WRITE)), "PUT")).Queries("versioning", "") // ListObjectsV2 - bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectsV2Handler, ACTION_LIST)), "LIST")).Queries("list-type", "2") + bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectsV2Handler, ACTION_LIST)), "LIST")).Queries("list-type", "2") // buckets with query // PutBucketOwnershipControls - bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.PutBucketOwnershipControls, ACTION_ADMIN), "PUT")).Queries("ownershipControls", "") + bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.PutBucketOwnershipControls, ACTION_ADMIN), "PUT")).Queries("ownershipControls", "") //GetBucketOwnershipControls - bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.GetBucketOwnershipControls, ACTION_READ), "GET")).Queries("ownershipControls", "") + bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.GetBucketOwnershipControls, ACTION_READ), "GET")).Queries("ownershipControls", "") //DeleteBucketOwnershipControls - bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteBucketOwnershipControls, ACTION_ADMIN), "DELETE")).Queries("ownershipControls", "") + bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.DeleteBucketOwnershipControls, ACTION_ADMIN), "DELETE")).Queries("ownershipControls", "") // raw buckets // PostPolicy - bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PostPolicyBucketHandler, ACTION_WRITE)), "POST")) + bucket.Methods(http.MethodPost).HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PostPolicyBucketHandler, ACTION_WRITE)), "POST")) // HeadBucket - bucket.Methods("HEAD").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.HeadBucketHandler, ACTION_READ)), "GET")) + bucket.Methods(http.MethodHead).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.HeadBucketHandler, ACTION_READ)), "GET")) // PutBucket - bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketHandler, ACTION_ADMIN)), "PUT")) + bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketHandler, ACTION_ADMIN)), "PUT")) // DeleteBucket - bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketHandler, ACTION_DELETE_BUCKET)), "DELETE")) + bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketHandler, ACTION_DELETE_BUCKET)), "DELETE")) // ListObjectsV1 (Legacy) - bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectsV1Handler, ACTION_LIST)), "LIST")) + bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectsV1Handler, ACTION_LIST)), "LIST")) // raw buckets } // ListBuckets - apiRouter.Methods("GET").Path("/").HandlerFunc(track(s3a.ListBucketsHandler, "LIST")) + apiRouter.Methods(http.MethodGet).Path("/").HandlerFunc(track(s3a.ListBucketsHandler, "LIST")) // NotFound apiRouter.NotFoundHandler = http.HandlerFunc(s3err.NotFoundHandler) diff --git a/weed/s3api/s3api_xsd_generated.go b/weed/s3api/s3api_xsd_generated.go index dd6a32ff2..f883287d5 100644 --- a/weed/s3api/s3api_xsd_generated.go +++ b/weed/s3api/s3api_xsd_generated.go @@ -1,3 +1,5 @@ +// Code generated by xsdgen. DO NOT EDIT. + package s3api import ( @@ -17,11 +19,546 @@ type AccessControlPolicy struct { } type AmazonCustomerByEmail struct { - EmailAddress string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ EmailAddress"` + EmailAddress string `xml:"EmailAddress"` +} + +type Anon1 struct { + Bucket string `xml:"Bucket"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` +} + +func (t *Anon1) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + type T Anon1 + var layout struct { + *T + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + layout.T = (*T)(t) + layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) + return e.EncodeElement(layout, start) +} +func (t *Anon1) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + type T Anon1 + var overlay struct { + *T + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + overlay.T = (*T)(t) + overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) + return d.DecodeElement(&overlay, &start) +} + +type Anon10 struct { +} + +type Anon11 struct { + Bucket string `xml:"Bucket"` + AccessControlList AccessControlList `xml:"AccessControlList,omitempty"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` +} + +func (t *Anon11) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + type T Anon11 + var layout struct { + *T + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + layout.T = (*T)(t) + layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) + return e.EncodeElement(layout, start) +} +func (t *Anon11) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + type T Anon11 + var overlay struct { + *T + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + overlay.T = (*T)(t) + overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) + return d.DecodeElement(&overlay, &start) +} + +type Anon12 struct { +} + +type Anon13 struct { + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + GetMetadata bool `xml:"GetMetadata"` + GetData bool `xml:"GetData"` + InlineData bool `xml:"InlineData"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` +} + +func (t *Anon13) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + type T Anon13 + var layout struct { + *T + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + layout.T = (*T)(t) + layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) + return e.EncodeElement(layout, start) +} +func (t *Anon13) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + type T Anon13 + var overlay struct { + *T + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + overlay.T = (*T)(t) + overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) + return d.DecodeElement(&overlay, &start) +} + +type Anon14 struct { + GetObjectResponse GetObjectResult `xml:"GetObjectResponse"` +} + +type Anon15 struct { + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + GetMetadata bool `xml:"GetMetadata"` + GetData bool `xml:"GetData"` + InlineData bool `xml:"InlineData"` + ByteRangeStart int64 `xml:"ByteRangeStart,omitempty"` + ByteRangeEnd int64 `xml:"ByteRangeEnd,omitempty"` + IfModifiedSince time.Time `xml:"IfModifiedSince,omitempty"` + IfUnmodifiedSince time.Time `xml:"IfUnmodifiedSince,omitempty"` + IfMatch []string `xml:"IfMatch,omitempty"` + IfNoneMatch []string `xml:"IfNoneMatch,omitempty"` + ReturnCompleteObjectOnConditionFailure bool `xml:"ReturnCompleteObjectOnConditionFailure,omitempty"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` +} + +func (t *Anon15) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + type T Anon15 + var layout struct { + *T + IfModifiedSince *xsdDateTime `xml:"IfModifiedSince,omitempty"` + IfUnmodifiedSince *xsdDateTime `xml:"IfUnmodifiedSince,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + layout.T = (*T)(t) + layout.IfModifiedSince = (*xsdDateTime)(&layout.T.IfModifiedSince) + layout.IfUnmodifiedSince = (*xsdDateTime)(&layout.T.IfUnmodifiedSince) + layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) + return e.EncodeElement(layout, start) +} +func (t *Anon15) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + type T Anon15 + var overlay struct { + *T + IfModifiedSince *xsdDateTime `xml:"IfModifiedSince,omitempty"` + IfUnmodifiedSince *xsdDateTime `xml:"IfUnmodifiedSince,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + overlay.T = (*T)(t) + overlay.IfModifiedSince = (*xsdDateTime)(&overlay.T.IfModifiedSince) + overlay.IfUnmodifiedSince = (*xsdDateTime)(&overlay.T.IfUnmodifiedSince) + overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) + return d.DecodeElement(&overlay, &start) +} + +type Anon16 struct { + GetObjectResponse GetObjectResult `xml:"GetObjectResponse"` +} + +type Anon17 struct { + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + Metadata []MetadataEntry `xml:"Metadata,omitempty"` + ContentLength int64 `xml:"ContentLength"` + AccessControlList AccessControlList `xml:"AccessControlList,omitempty"` + StorageClass StorageClass `xml:"StorageClass,omitempty"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` +} + +func (t *Anon17) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + type T Anon17 + var layout struct { + *T + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + layout.T = (*T)(t) + layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) + return e.EncodeElement(layout, start) +} +func (t *Anon17) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + type T Anon17 + var overlay struct { + *T + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + overlay.T = (*T)(t) + overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) + return d.DecodeElement(&overlay, &start) +} + +type Anon18 struct { + PutObjectResponse PutObjectResult `xml:"PutObjectResponse"` +} + +type Anon19 struct { + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + Metadata []MetadataEntry `xml:"Metadata,omitempty"` + Data []byte `xml:"Data"` + ContentLength int64 `xml:"ContentLength"` + AccessControlList AccessControlList `xml:"AccessControlList,omitempty"` + StorageClass StorageClass `xml:"StorageClass,omitempty"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` +} + +func (t *Anon19) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + type T Anon19 + var layout struct { + *T + Data *xsdBase64Binary `xml:"Data"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + layout.T = (*T)(t) + layout.Data = (*xsdBase64Binary)(&layout.T.Data) + layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) + return e.EncodeElement(layout, start) +} +func (t *Anon19) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + type T Anon19 + var overlay struct { + *T + Data *xsdBase64Binary `xml:"Data"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + overlay.T = (*T)(t) + overlay.Data = (*xsdBase64Binary)(&overlay.T.Data) + overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) + return d.DecodeElement(&overlay, &start) +} + +type Anon2 struct { + GetBucketLoggingStatusResponse BucketLoggingStatus `xml:"GetBucketLoggingStatusResponse"` +} + +type Anon20 struct { + PutObjectInlineResponse PutObjectResult `xml:"PutObjectInlineResponse"` +} + +type Anon21 struct { + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` +} + +func (t *Anon21) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + type T Anon21 + var layout struct { + *T + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + layout.T = (*T)(t) + layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) + return e.EncodeElement(layout, start) +} +func (t *Anon21) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + type T Anon21 + var overlay struct { + *T + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + overlay.T = (*T)(t) + overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) + return d.DecodeElement(&overlay, &start) +} + +type Anon22 struct { + DeleteObjectResponse Status `xml:"DeleteObjectResponse"` +} + +type Anon23 struct { + Bucket string `xml:"Bucket"` + Prefix string `xml:"Prefix,omitempty"` + Marker string `xml:"Marker,omitempty"` + MaxKeys int `xml:"MaxKeys,omitempty"` + Delimiter string `xml:"Delimiter,omitempty"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` +} + +func (t *Anon23) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + type T Anon23 + var layout struct { + *T + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + layout.T = (*T)(t) + layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) + return e.EncodeElement(layout, start) +} +func (t *Anon23) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + type T Anon23 + var overlay struct { + *T + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + overlay.T = (*T)(t) + overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) + return d.DecodeElement(&overlay, &start) +} + +type Anon24 struct { + ListBucketResponse ListBucketResult `xml:"ListBucketResponse"` +} + +type Anon25 struct { + ListVersionsResponse ListVersionsResult `xml:"ListVersionsResponse"` +} + +type Anon26 struct { + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` +} + +func (t *Anon26) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + type T Anon26 + var layout struct { + *T + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + layout.T = (*T)(t) + layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) + return e.EncodeElement(layout, start) +} +func (t *Anon26) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + type T Anon26 + var overlay struct { + *T + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + overlay.T = (*T)(t) + overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) + return d.DecodeElement(&overlay, &start) +} + +type Anon27 struct { + ListAllMyBucketsResponse ListAllMyBucketsResult `xml:"ListAllMyBucketsResponse"` +} + +type Anon28 struct { + Location string `xml:"Location"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + ETag string `xml:"ETag"` +} + +type Anon29 struct { + SourceBucket string `xml:"SourceBucket"` + SourceKey string `xml:"SourceKey"` + DestinationBucket string `xml:"DestinationBucket"` + DestinationKey string `xml:"DestinationKey"` + MetadataDirective MetadataDirective `xml:"MetadataDirective,omitempty"` + Metadata []MetadataEntry `xml:"Metadata,omitempty"` + AccessControlList AccessControlList `xml:"AccessControlList,omitempty"` + CopySourceIfModifiedSince time.Time `xml:"CopySourceIfModifiedSince,omitempty"` + CopySourceIfUnmodifiedSince time.Time `xml:"CopySourceIfUnmodifiedSince,omitempty"` + CopySourceIfMatch []string `xml:"CopySourceIfMatch,omitempty"` + CopySourceIfNoneMatch []string `xml:"CopySourceIfNoneMatch,omitempty"` + StorageClass StorageClass `xml:"StorageClass,omitempty"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` +} + +func (t *Anon29) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + type T Anon29 + var layout struct { + *T + CopySourceIfModifiedSince *xsdDateTime `xml:"CopySourceIfModifiedSince,omitempty"` + CopySourceIfUnmodifiedSince *xsdDateTime `xml:"CopySourceIfUnmodifiedSince,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + layout.T = (*T)(t) + layout.CopySourceIfModifiedSince = (*xsdDateTime)(&layout.T.CopySourceIfModifiedSince) + layout.CopySourceIfUnmodifiedSince = (*xsdDateTime)(&layout.T.CopySourceIfUnmodifiedSince) + layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) + return e.EncodeElement(layout, start) +} +func (t *Anon29) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + type T Anon29 + var overlay struct { + *T + CopySourceIfModifiedSince *xsdDateTime `xml:"CopySourceIfModifiedSince,omitempty"` + CopySourceIfUnmodifiedSince *xsdDateTime `xml:"CopySourceIfUnmodifiedSince,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + overlay.T = (*T)(t) + overlay.CopySourceIfModifiedSince = (*xsdDateTime)(&overlay.T.CopySourceIfModifiedSince) + overlay.CopySourceIfUnmodifiedSince = (*xsdDateTime)(&overlay.T.CopySourceIfUnmodifiedSince) + overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) + return d.DecodeElement(&overlay, &start) +} + +type Anon3 struct { + Bucket string `xml:"Bucket"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` + BucketLoggingStatus BucketLoggingStatus `xml:"BucketLoggingStatus"` +} + +func (t *Anon3) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + type T Anon3 + var layout struct { + *T + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + layout.T = (*T)(t) + layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) + return e.EncodeElement(layout, start) +} +func (t *Anon3) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + type T Anon3 + var overlay struct { + *T + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + overlay.T = (*T)(t) + overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) + return d.DecodeElement(&overlay, &start) +} + +type Anon30 struct { + CopyObjectResult CopyObjectResult `xml:"CopyObjectResult"` +} + +type Anon4 struct { +} + +type Anon5 struct { + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` +} + +func (t *Anon5) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + type T Anon5 + var layout struct { + *T + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + layout.T = (*T)(t) + layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) + return e.EncodeElement(layout, start) +} +func (t *Anon5) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + type T Anon5 + var overlay struct { + *T + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + overlay.T = (*T)(t) + overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) + return d.DecodeElement(&overlay, &start) +} + +type Anon6 struct { + GetObjectAccessControlPolicyResponse AccessControlPolicy `xml:"GetObjectAccessControlPolicyResponse"` +} + +type Anon7 struct { + Bucket string `xml:"Bucket"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` +} + +func (t *Anon7) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + type T Anon7 + var layout struct { + *T + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + layout.T = (*T)(t) + layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) + return e.EncodeElement(layout, start) +} +func (t *Anon7) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + type T Anon7 + var overlay struct { + *T + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + overlay.T = (*T)(t) + overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) + return d.DecodeElement(&overlay, &start) +} + +type Anon8 struct { + GetBucketAccessControlPolicyResponse AccessControlPolicy `xml:"GetBucketAccessControlPolicyResponse"` +} + +type Anon9 struct { + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + AccessControlList AccessControlList `xml:"AccessControlList"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` +} + +func (t *Anon9) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + type T Anon9 + var layout struct { + *T + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + layout.T = (*T)(t) + layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) + return e.EncodeElement(layout, start) +} +func (t *Anon9) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + type T Anon9 + var overlay struct { + *T + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` + } + overlay.T = (*T)(t) + overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) + return d.DecodeElement(&overlay, &start) } type BucketLoggingStatus struct { - LoggingEnabled LoggingSettings `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LoggingEnabled,omitempty"` + LoggingEnabled LoggingSettings `xml:"LoggingEnabled,omitempty"` } type CanonicalUser struct { @@ -30,31 +567,31 @@ type CanonicalUser struct { } type CopyObject struct { - SourceBucket string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ SourceBucket"` - SourceKey string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ SourceKey"` - DestinationBucket string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DestinationBucket"` - DestinationKey string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DestinationKey"` - MetadataDirective MetadataDirective `xml:"http://s3.amazonaws.com/doc/2006-03-01/ MetadataDirective,omitempty"` - Metadata []MetadataEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Metadata,omitempty"` - AccessControlList AccessControlList `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlList,omitempty"` - CopySourceIfModifiedSince time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopySourceIfModifiedSince,omitempty"` - CopySourceIfUnmodifiedSince time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopySourceIfUnmodifiedSince,omitempty"` - CopySourceIfMatch []string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopySourceIfMatch,omitempty"` - CopySourceIfNoneMatch []string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopySourceIfNoneMatch,omitempty"` - StorageClass StorageClass `xml:"http://s3.amazonaws.com/doc/2006-03-01/ StorageClass,omitempty"` - AWSAccessKeyId string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AWSAccessKeyId,omitempty"` - Timestamp time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` - Signature string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Signature,omitempty"` - Credential string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Credential,omitempty"` + SourceBucket string `xml:"SourceBucket"` + SourceKey string `xml:"SourceKey"` + DestinationBucket string `xml:"DestinationBucket"` + DestinationKey string `xml:"DestinationKey"` + MetadataDirective MetadataDirective `xml:"MetadataDirective,omitempty"` + Metadata []MetadataEntry `xml:"Metadata,omitempty"` + AccessControlList AccessControlList `xml:"AccessControlList,omitempty"` + CopySourceIfModifiedSince time.Time `xml:"CopySourceIfModifiedSince,omitempty"` + CopySourceIfUnmodifiedSince time.Time `xml:"CopySourceIfUnmodifiedSince,omitempty"` + CopySourceIfMatch []string `xml:"CopySourceIfMatch,omitempty"` + CopySourceIfNoneMatch []string `xml:"CopySourceIfNoneMatch,omitempty"` + StorageClass StorageClass `xml:"StorageClass,omitempty"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` } func (t *CopyObject) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type T CopyObject var layout struct { *T - CopySourceIfModifiedSince *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopySourceIfModifiedSince,omitempty"` - CopySourceIfUnmodifiedSince *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopySourceIfUnmodifiedSince,omitempty"` - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + CopySourceIfModifiedSince *xsdDateTime `xml:"CopySourceIfModifiedSince,omitempty"` + CopySourceIfUnmodifiedSince *xsdDateTime `xml:"CopySourceIfUnmodifiedSince,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } layout.T = (*T)(t) layout.CopySourceIfModifiedSince = (*xsdDateTime)(&layout.T.CopySourceIfModifiedSince) @@ -66,9 +603,9 @@ func (t *CopyObject) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error type T CopyObject var overlay struct { *T - CopySourceIfModifiedSince *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopySourceIfModifiedSince,omitempty"` - CopySourceIfUnmodifiedSince *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopySourceIfUnmodifiedSince,omitempty"` - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + CopySourceIfModifiedSince *xsdDateTime `xml:"CopySourceIfModifiedSince,omitempty"` + CopySourceIfUnmodifiedSince *xsdDateTime `xml:"CopySourceIfUnmodifiedSince,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } overlay.T = (*T)(t) overlay.CopySourceIfModifiedSince = (*xsdDateTime)(&overlay.T.CopySourceIfModifiedSince) @@ -78,19 +615,19 @@ func (t *CopyObject) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error } type CopyObjectResponse struct { - CopyObjectResult CopyObjectResult `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult"` + CopyObjectResult CopyObjectResult `xml:"CopyObjectResult"` } type CopyObjectResult struct { - LastModified time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"` - ETag string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ETag"` + LastModified time.Time `xml:"LastModified"` + ETag string `xml:"ETag"` } func (t *CopyObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type T CopyObjectResult var layout struct { *T - LastModified *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"` + LastModified *xsdDateTime `xml:"LastModified"` } layout.T = (*T)(t) layout.LastModified = (*xsdDateTime)(&layout.T.LastModified) @@ -100,7 +637,7 @@ func (t *CopyObjectResult) UnmarshalXML(d *xml.Decoder, start xml.StartElement) type T CopyObjectResult var overlay struct { *T - LastModified *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"` + LastModified *xsdDateTime `xml:"LastModified"` } overlay.T = (*T)(t) overlay.LastModified = (*xsdDateTime)(&overlay.T.LastModified) @@ -108,18 +645,18 @@ func (t *CopyObjectResult) UnmarshalXML(d *xml.Decoder, start xml.StartElement) } type CreateBucket struct { - Bucket string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Bucket"` - AccessControlList AccessControlList `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlList,omitempty"` - AWSAccessKeyId string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AWSAccessKeyId,omitempty"` - Timestamp time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` - Signature string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Signature,omitempty"` + Bucket string `xml:"Bucket"` + AccessControlList AccessControlList `xml:"AccessControlList,omitempty"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` } func (t *CreateBucket) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type T CreateBucket var layout struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } layout.T = (*T)(t) layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) @@ -129,7 +666,7 @@ func (t *CreateBucket) UnmarshalXML(d *xml.Decoder, start xml.StartElement) erro type T CreateBucket var overlay struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } overlay.T = (*T)(t) overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) @@ -137,30 +674,30 @@ func (t *CreateBucket) UnmarshalXML(d *xml.Decoder, start xml.StartElement) erro } type CreateBucketConfiguration struct { - LocationConstraint string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LocationConstraint"` + LocationConstraint string `xml:"LocationConstraint"` } type CreateBucketResponse struct { - CreateBucketReturn CreateBucketResult `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreateBucketReturn"` + CreateBucketReturn CreateBucketResult `xml:"CreateBucketReturn"` } type CreateBucketResult struct { - BucketName string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ BucketName"` + BucketName string `xml:"BucketName"` } type DeleteBucket struct { - Bucket string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Bucket"` - AWSAccessKeyId string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AWSAccessKeyId,omitempty"` - Timestamp time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` - Signature string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Signature,omitempty"` - Credential string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Credential,omitempty"` + Bucket string `xml:"Bucket"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` } func (t *DeleteBucket) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type T DeleteBucket var layout struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } layout.T = (*T)(t) layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) @@ -170,7 +707,7 @@ func (t *DeleteBucket) UnmarshalXML(d *xml.Decoder, start xml.StartElement) erro type T DeleteBucket var overlay struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } overlay.T = (*T)(t) overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) @@ -178,22 +715,22 @@ func (t *DeleteBucket) UnmarshalXML(d *xml.Decoder, start xml.StartElement) erro } type DeleteBucketResponse struct { - DeleteBucketResponse Status `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DeleteBucketResponse"` + DeleteBucketResponse Status `xml:"DeleteBucketResponse"` } type DeleteMarkerEntry struct { - Key string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Key"` - VersionId string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ VersionId"` - IsLatest bool `xml:"http://s3.amazonaws.com/doc/2006-03-01/ IsLatest"` - LastModified time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"` - Owner CanonicalUser `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Owner,omitempty"` + Key string `xml:"Key"` + VersionId string `xml:"VersionId"` + IsLatest bool `xml:"IsLatest"` + LastModified time.Time `xml:"LastModified"` + Owner CanonicalUser `xml:"Owner,omitempty"` } func (t *DeleteMarkerEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type T DeleteMarkerEntry var layout struct { *T - LastModified *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"` + LastModified *xsdDateTime `xml:"LastModified"` } layout.T = (*T)(t) layout.LastModified = (*xsdDateTime)(&layout.T.LastModified) @@ -203,7 +740,7 @@ func (t *DeleteMarkerEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElement) type T DeleteMarkerEntry var overlay struct { *T - LastModified *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"` + LastModified *xsdDateTime `xml:"LastModified"` } overlay.T = (*T)(t) overlay.LastModified = (*xsdDateTime)(&overlay.T.LastModified) @@ -211,19 +748,19 @@ func (t *DeleteMarkerEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElement) } type DeleteObject struct { - Bucket string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Bucket"` - Key string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Key"` - AWSAccessKeyId string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AWSAccessKeyId,omitempty"` - Timestamp time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` - Signature string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Signature,omitempty"` - Credential string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Credential,omitempty"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` } func (t *DeleteObject) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type T DeleteObject var layout struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } layout.T = (*T)(t) layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) @@ -233,7 +770,7 @@ func (t *DeleteObject) UnmarshalXML(d *xml.Decoder, start xml.StartElement) erro type T DeleteObject var overlay struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } overlay.T = (*T)(t) overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) @@ -241,22 +778,22 @@ func (t *DeleteObject) UnmarshalXML(d *xml.Decoder, start xml.StartElement) erro } type DeleteObjectResponse struct { - DeleteObjectResponse Status `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DeleteObjectResponse"` + DeleteObjectResponse Status `xml:"DeleteObjectResponse"` } type GetBucketAccessControlPolicy struct { - Bucket string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Bucket"` - AWSAccessKeyId string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AWSAccessKeyId,omitempty"` - Timestamp time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` - Signature string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Signature,omitempty"` - Credential string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Credential,omitempty"` + Bucket string `xml:"Bucket"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` } func (t *GetBucketAccessControlPolicy) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type T GetBucketAccessControlPolicy var layout struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } layout.T = (*T)(t) layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) @@ -266,7 +803,7 @@ func (t *GetBucketAccessControlPolicy) UnmarshalXML(d *xml.Decoder, start xml.St type T GetBucketAccessControlPolicy var overlay struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } overlay.T = (*T)(t) overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) @@ -274,22 +811,22 @@ func (t *GetBucketAccessControlPolicy) UnmarshalXML(d *xml.Decoder, start xml.St } type GetBucketAccessControlPolicyResponse struct { - GetBucketAccessControlPolicyResponse AccessControlPolicy `xml:"http://s3.amazonaws.com/doc/2006-03-01/ GetBucketAccessControlPolicyResponse"` + GetBucketAccessControlPolicyResponse AccessControlPolicy `xml:"GetBucketAccessControlPolicyResponse"` } type GetBucketLoggingStatus struct { - Bucket string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Bucket"` - AWSAccessKeyId string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AWSAccessKeyId,omitempty"` - Timestamp time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` - Signature string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Signature,omitempty"` - Credential string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Credential,omitempty"` + Bucket string `xml:"Bucket"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` } func (t *GetBucketLoggingStatus) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type T GetBucketLoggingStatus var layout struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } layout.T = (*T)(t) layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) @@ -299,7 +836,7 @@ func (t *GetBucketLoggingStatus) UnmarshalXML(d *xml.Decoder, start xml.StartEle type T GetBucketLoggingStatus var overlay struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } overlay.T = (*T)(t) overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) @@ -307,26 +844,26 @@ func (t *GetBucketLoggingStatus) UnmarshalXML(d *xml.Decoder, start xml.StartEle } type GetBucketLoggingStatusResponse struct { - GetBucketLoggingStatusResponse BucketLoggingStatus `xml:"http://s3.amazonaws.com/doc/2006-03-01/ GetBucketLoggingStatusResponse"` + GetBucketLoggingStatusResponse BucketLoggingStatus `xml:"GetBucketLoggingStatusResponse"` } type GetObject struct { - Bucket string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Bucket"` - Key string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Key"` - GetMetadata bool `xml:"http://s3.amazonaws.com/doc/2006-03-01/ GetMetadata"` - GetData bool `xml:"http://s3.amazonaws.com/doc/2006-03-01/ GetData"` - InlineData bool `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InlineData"` - AWSAccessKeyId string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AWSAccessKeyId,omitempty"` - Timestamp time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` - Signature string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Signature,omitempty"` - Credential string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Credential,omitempty"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + GetMetadata bool `xml:"GetMetadata"` + GetData bool `xml:"GetData"` + InlineData bool `xml:"InlineData"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` } func (t *GetObject) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type T GetObject var layout struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } layout.T = (*T)(t) layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) @@ -336,7 +873,7 @@ func (t *GetObject) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { type T GetObject var overlay struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } overlay.T = (*T)(t) overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) @@ -344,19 +881,19 @@ func (t *GetObject) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { } type GetObjectAccessControlPolicy struct { - Bucket string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Bucket"` - Key string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Key"` - AWSAccessKeyId string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AWSAccessKeyId,omitempty"` - Timestamp time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` - Signature string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Signature,omitempty"` - Credential string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Credential,omitempty"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` } func (t *GetObjectAccessControlPolicy) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type T GetObjectAccessControlPolicy var layout struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } layout.T = (*T)(t) layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) @@ -366,7 +903,7 @@ func (t *GetObjectAccessControlPolicy) UnmarshalXML(d *xml.Decoder, start xml.St type T GetObjectAccessControlPolicy var overlay struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } overlay.T = (*T)(t) overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) @@ -374,35 +911,35 @@ func (t *GetObjectAccessControlPolicy) UnmarshalXML(d *xml.Decoder, start xml.St } type GetObjectAccessControlPolicyResponse struct { - GetObjectAccessControlPolicyResponse AccessControlPolicy `xml:"http://s3.amazonaws.com/doc/2006-03-01/ GetObjectAccessControlPolicyResponse"` + GetObjectAccessControlPolicyResponse AccessControlPolicy `xml:"GetObjectAccessControlPolicyResponse"` } type GetObjectExtended struct { - Bucket string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Bucket"` - Key string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Key"` - GetMetadata bool `xml:"http://s3.amazonaws.com/doc/2006-03-01/ GetMetadata"` - GetData bool `xml:"http://s3.amazonaws.com/doc/2006-03-01/ GetData"` - InlineData bool `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InlineData"` - ByteRangeStart int64 `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ByteRangeStart,omitempty"` - ByteRangeEnd int64 `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ByteRangeEnd,omitempty"` - IfModifiedSince time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ IfModifiedSince,omitempty"` - IfUnmodifiedSince time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ IfUnmodifiedSince,omitempty"` - IfMatch []string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ IfMatch,omitempty"` - IfNoneMatch []string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ IfNoneMatch,omitempty"` - ReturnCompleteObjectOnConditionFailure bool `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ReturnCompleteObjectOnConditionFailure,omitempty"` - AWSAccessKeyId string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AWSAccessKeyId,omitempty"` - Timestamp time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` - Signature string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Signature,omitempty"` - Credential string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Credential,omitempty"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + GetMetadata bool `xml:"GetMetadata"` + GetData bool `xml:"GetData"` + InlineData bool `xml:"InlineData"` + ByteRangeStart int64 `xml:"ByteRangeStart,omitempty"` + ByteRangeEnd int64 `xml:"ByteRangeEnd,omitempty"` + IfModifiedSince time.Time `xml:"IfModifiedSince,omitempty"` + IfUnmodifiedSince time.Time `xml:"IfUnmodifiedSince,omitempty"` + IfMatch []string `xml:"IfMatch,omitempty"` + IfNoneMatch []string `xml:"IfNoneMatch,omitempty"` + ReturnCompleteObjectOnConditionFailure bool `xml:"ReturnCompleteObjectOnConditionFailure,omitempty"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` } func (t *GetObjectExtended) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type T GetObjectExtended var layout struct { *T - IfModifiedSince *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ IfModifiedSince,omitempty"` - IfUnmodifiedSince *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ IfUnmodifiedSince,omitempty"` - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + IfModifiedSince *xsdDateTime `xml:"IfModifiedSince,omitempty"` + IfUnmodifiedSince *xsdDateTime `xml:"IfUnmodifiedSince,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } layout.T = (*T)(t) layout.IfModifiedSince = (*xsdDateTime)(&layout.T.IfModifiedSince) @@ -414,9 +951,9 @@ func (t *GetObjectExtended) UnmarshalXML(d *xml.Decoder, start xml.StartElement) type T GetObjectExtended var overlay struct { *T - IfModifiedSince *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ IfModifiedSince,omitempty"` - IfUnmodifiedSince *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ IfUnmodifiedSince,omitempty"` - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + IfModifiedSince *xsdDateTime `xml:"IfModifiedSince,omitempty"` + IfUnmodifiedSince *xsdDateTime `xml:"IfUnmodifiedSince,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } overlay.T = (*T)(t) overlay.IfModifiedSince = (*xsdDateTime)(&overlay.T.IfModifiedSince) @@ -426,27 +963,27 @@ func (t *GetObjectExtended) UnmarshalXML(d *xml.Decoder, start xml.StartElement) } type GetObjectExtendedResponse struct { - GetObjectResponse GetObjectResult `xml:"http://s3.amazonaws.com/doc/2006-03-01/ GetObjectResponse"` + GetObjectResponse GetObjectResult `xml:"GetObjectResponse"` } type GetObjectResponse struct { - GetObjectResponse GetObjectResult `xml:"http://s3.amazonaws.com/doc/2006-03-01/ GetObjectResponse"` + GetObjectResponse GetObjectResult `xml:"GetObjectResponse"` } type GetObjectResult struct { - Metadata []MetadataEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Metadata,omitempty"` - Data []byte `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Data,omitempty"` - LastModified time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"` - ETag string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ETag"` - Status Status `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Status"` + Status Status `xml:"Status"` + Metadata []MetadataEntry `xml:"Metadata,omitempty"` + Data []byte `xml:"Data,omitempty"` + LastModified time.Time `xml:"LastModified"` + ETag string `xml:"ETag"` } func (t *GetObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type T GetObjectResult var layout struct { *T - Data *xsdBase64Binary `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Data,omitempty"` - LastModified *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"` + Data *xsdBase64Binary `xml:"Data,omitempty"` + LastModified *xsdDateTime `xml:"LastModified"` } layout.T = (*T)(t) layout.Data = (*xsdBase64Binary)(&layout.T.Data) @@ -457,8 +994,8 @@ func (t *GetObjectResult) UnmarshalXML(d *xml.Decoder, start xml.StartElement) e type T GetObjectResult var overlay struct { *T - Data *xsdBase64Binary `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Data,omitempty"` - LastModified *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"` + Data *xsdBase64Binary `xml:"Data,omitempty"` + LastModified *xsdDateTime `xml:"LastModified"` } overlay.T = (*T)(t) overlay.Data = (*xsdBase64Binary)(&overlay.T.Data) @@ -471,30 +1008,21 @@ type Grant struct { Permission Permission `xml:"Permission"` } -type Grantee struct { - XMLNS string `xml:"xmlns:xsi,attr"` - XMLXSI string `xml:"xsi:type,attr"` - Type string `xml:"Type"` - ID string `xml:"ID,omitempty"` - DisplayName string `xml:"DisplayName,omitempty"` - URI string `xml:"URI,omitempty"` -} - type Group struct { - URI string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ URI"` + URI string `xml:"URI"` } type ListAllMyBuckets struct { - AWSAccessKeyId string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AWSAccessKeyId,omitempty"` - Timestamp time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` - Signature string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Signature,omitempty"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` } func (t *ListAllMyBuckets) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type T ListAllMyBuckets var layout struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } layout.T = (*T)(t) layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) @@ -504,7 +1032,7 @@ func (t *ListAllMyBuckets) UnmarshalXML(d *xml.Decoder, start xml.StartElement) type T ListAllMyBuckets var overlay struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } overlay.T = (*T)(t) overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) @@ -542,26 +1070,31 @@ type ListAllMyBucketsList struct { } type ListAllMyBucketsResponse struct { - ListAllMyBucketsResponse ListAllMyBucketsResult `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResponse"` + ListAllMyBucketsResponse ListAllMyBucketsResult `xml:"ListAllMyBucketsResponse"` +} + +type ListAllMyBucketsResult struct { + Owner CanonicalUser `xml:"Owner"` + Buckets ListAllMyBucketsList `xml:"Buckets"` } type ListBucket struct { - Bucket string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Bucket"` - Prefix string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Prefix,omitempty"` - Marker string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Marker,omitempty"` - MaxKeys int `xml:"http://s3.amazonaws.com/doc/2006-03-01/ MaxKeys,omitempty"` - Delimiter string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Delimiter,omitempty"` - AWSAccessKeyId string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AWSAccessKeyId,omitempty"` - Timestamp time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` - Signature string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Signature,omitempty"` - Credential string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Credential,omitempty"` + Bucket string `xml:"Bucket"` + Prefix string `xml:"Prefix,omitempty"` + Marker string `xml:"Marker,omitempty"` + MaxKeys int `xml:"MaxKeys,omitempty"` + Delimiter string `xml:"Delimiter,omitempty"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` } func (t *ListBucket) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type T ListBucket var layout struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } layout.T = (*T)(t) layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) @@ -571,7 +1104,7 @@ func (t *ListBucket) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error type T ListBucket var overlay struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } overlay.T = (*T)(t) overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) @@ -579,11 +1112,10 @@ func (t *ListBucket) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error } type ListBucketResponse struct { - ListBucketResponse ListBucketResult `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResponse"` + ListBucketResponse ListBucketResult `xml:"ListBucketResponse"` } type ListBucketResult struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"` Metadata []MetadataEntry `xml:"Metadata,omitempty"` Name string `xml:"Name"` Prefix string `xml:"Prefix"` @@ -594,6 +1126,7 @@ type ListBucketResult struct { IsTruncated bool `xml:"IsTruncated"` Contents []ListEntry `xml:"Contents,omitempty"` CommonPrefixes []PrefixEntry `xml:"CommonPrefixes,omitempty"` + EncodingType string `xml:"EncodingType"` } type ListEntry struct { @@ -627,48 +1160,44 @@ func (t *ListEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { } type ListVersionsResponse struct { - ListVersionsResponse ListVersionsResult `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListVersionsResponse"` + ListVersionsResponse ListVersionsResult `xml:"ListVersionsResponse"` } type ListVersionsResult struct { - Metadata []MetadataEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Metadata,omitempty"` - Name string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Name"` - Prefix string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Prefix"` - KeyMarker string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ KeyMarker"` - VersionIdMarker string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ VersionIdMarker"` - NextKeyMarker string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ NextKeyMarker,omitempty"` - NextVersionIdMarker string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ NextVersionIdMarker,omitempty"` - MaxKeys int `xml:"http://s3.amazonaws.com/doc/2006-03-01/ MaxKeys"` - Delimiter string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Delimiter,omitempty"` - IsTruncated bool `xml:"http://s3.amazonaws.com/doc/2006-03-01/ IsTruncated"` - Version VersionEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Version"` - DeleteMarker DeleteMarkerEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DeleteMarker"` - CommonPrefixes []PrefixEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CommonPrefixes,omitempty"` -} - -type LocationConstraint struct { - LocationConstraint string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LocationConstraint"` + Metadata []MetadataEntry `xml:"Metadata,omitempty"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix"` + KeyMarker string `xml:"KeyMarker"` + VersionIdMarker string `xml:"VersionIdMarker"` + NextKeyMarker string `xml:"NextKeyMarker,omitempty"` + NextVersionIdMarker string `xml:"NextVersionIdMarker,omitempty"` + MaxKeys int `xml:"MaxKeys"` + Delimiter string `xml:"Delimiter,omitempty"` + IsTruncated bool `xml:"IsTruncated"` + Version VersionEntry `xml:"Version,omitempty"` + DeleteMarker DeleteMarkerEntry `xml:"DeleteMarker,omitempty"` + CommonPrefixes []PrefixEntry `xml:"CommonPrefixes,omitempty"` } type LoggingSettings struct { - TargetBucket string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ TargetBucket"` - TargetPrefix string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ TargetPrefix"` - TargetGrants AccessControlList `xml:"http://s3.amazonaws.com/doc/2006-03-01/ TargetGrants,omitempty"` + TargetBucket string `xml:"TargetBucket"` + TargetPrefix string `xml:"TargetPrefix"` + TargetGrants AccessControlList `xml:"TargetGrants,omitempty"` } // May be one of COPY, REPLACE type MetadataDirective string type MetadataEntry struct { - Name string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Name"` - Value string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Value"` + Name string `xml:"Name"` + Value string `xml:"Value"` } // May be one of Enabled, Disabled type MfaDeleteStatus string type NotificationConfiguration struct { - TopicConfiguration []TopicConfiguration `xml:"http://s3.amazonaws.com/doc/2006-03-01/ TopicConfiguration,omitempty"` + TopicConfiguration []TopicConfiguration `xml:"TopicConfiguration,omitempty"` } // May be one of BucketOwner, Requester @@ -678,10 +1207,10 @@ type Payer string type Permission string type PostResponse struct { - Location string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Location"` - Bucket string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Bucket"` - Key string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Key"` - ETag string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ETag"` + Location string `xml:"Location"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + ETag string `xml:"ETag"` } type PrefixEntry struct { @@ -689,23 +1218,23 @@ type PrefixEntry struct { } type PutObject struct { - Bucket string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Bucket"` - Key string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Key"` - Metadata []MetadataEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Metadata,omitempty"` - ContentLength int64 `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ContentLength"` - AccessControlList AccessControlList `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlList,omitempty"` - StorageClass StorageClass `xml:"http://s3.amazonaws.com/doc/2006-03-01/ StorageClass,omitempty"` - AWSAccessKeyId string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AWSAccessKeyId,omitempty"` - Timestamp time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` - Signature string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Signature,omitempty"` - Credential string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Credential,omitempty"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + Metadata []MetadataEntry `xml:"Metadata,omitempty"` + ContentLength int64 `xml:"ContentLength"` + AccessControlList AccessControlList `xml:"AccessControlList,omitempty"` + StorageClass StorageClass `xml:"StorageClass,omitempty"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` } func (t *PutObject) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type T PutObject var layout struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } layout.T = (*T)(t) layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) @@ -715,7 +1244,7 @@ func (t *PutObject) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { type T PutObject var overlay struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } overlay.T = (*T)(t) overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) @@ -723,25 +1252,25 @@ func (t *PutObject) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { } type PutObjectInline struct { - Bucket string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Bucket"` - Key string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Key"` - Metadata []MetadataEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Metadata,omitempty"` - Data []byte `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Data"` - ContentLength int64 `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ContentLength"` - AccessControlList AccessControlList `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlList,omitempty"` - StorageClass StorageClass `xml:"http://s3.amazonaws.com/doc/2006-03-01/ StorageClass,omitempty"` - AWSAccessKeyId string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AWSAccessKeyId,omitempty"` - Timestamp time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` - Signature string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Signature,omitempty"` - Credential string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Credential,omitempty"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + Metadata []MetadataEntry `xml:"Metadata,omitempty"` + Data []byte `xml:"Data"` + ContentLength int64 `xml:"ContentLength"` + AccessControlList AccessControlList `xml:"AccessControlList,omitempty"` + StorageClass StorageClass `xml:"StorageClass,omitempty"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` } func (t *PutObjectInline) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type T PutObjectInline var layout struct { *T - Data *xsdBase64Binary `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Data"` - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Data *xsdBase64Binary `xml:"Data"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } layout.T = (*T)(t) layout.Data = (*xsdBase64Binary)(&layout.T.Data) @@ -752,8 +1281,8 @@ func (t *PutObjectInline) UnmarshalXML(d *xml.Decoder, start xml.StartElement) e type T PutObjectInline var overlay struct { *T - Data *xsdBase64Binary `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Data"` - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Data *xsdBase64Binary `xml:"Data"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } overlay.T = (*T)(t) overlay.Data = (*xsdBase64Binary)(&overlay.T.Data) @@ -762,23 +1291,23 @@ func (t *PutObjectInline) UnmarshalXML(d *xml.Decoder, start xml.StartElement) e } type PutObjectInlineResponse struct { - PutObjectInlineResponse PutObjectResult `xml:"http://s3.amazonaws.com/doc/2006-03-01/ PutObjectInlineResponse"` + PutObjectInlineResponse PutObjectResult `xml:"PutObjectInlineResponse"` } type PutObjectResponse struct { - PutObjectResponse PutObjectResult `xml:"http://s3.amazonaws.com/doc/2006-03-01/ PutObjectResponse"` + PutObjectResponse PutObjectResult `xml:"PutObjectResponse"` } type PutObjectResult struct { - ETag string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ETag"` - LastModified time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"` + ETag string `xml:"ETag"` + LastModified time.Time `xml:"LastModified"` } func (t *PutObjectResult) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type T PutObjectResult var layout struct { *T - LastModified *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"` + LastModified *xsdDateTime `xml:"LastModified"` } layout.T = (*T)(t) layout.LastModified = (*xsdDateTime)(&layout.T.LastModified) @@ -788,7 +1317,7 @@ func (t *PutObjectResult) UnmarshalXML(d *xml.Decoder, start xml.StartElement) e type T PutObjectResult var overlay struct { *T - LastModified *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"` + LastModified *xsdDateTime `xml:"LastModified"` } overlay.T = (*T)(t) overlay.LastModified = (*xsdDateTime)(&overlay.T.LastModified) @@ -796,27 +1325,27 @@ func (t *PutObjectResult) UnmarshalXML(d *xml.Decoder, start xml.StartElement) e } type RequestPaymentConfiguration struct { - Payer Payer `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Payer"` + Payer Payer `xml:"Payer"` } type Result struct { - Status Status `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Status"` + Status Status `xml:"Status"` } type SetBucketAccessControlPolicy struct { - Bucket string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Bucket"` - AccessControlList AccessControlList `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlList,omitempty"` - AWSAccessKeyId string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AWSAccessKeyId,omitempty"` - Timestamp time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` - Signature string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Signature,omitempty"` - Credential string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Credential,omitempty"` + Bucket string `xml:"Bucket"` + AccessControlList AccessControlList `xml:"AccessControlList,omitempty"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` } func (t *SetBucketAccessControlPolicy) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type T SetBucketAccessControlPolicy var layout struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } layout.T = (*T)(t) layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) @@ -826,7 +1355,7 @@ func (t *SetBucketAccessControlPolicy) UnmarshalXML(d *xml.Decoder, start xml.St type T SetBucketAccessControlPolicy var overlay struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } overlay.T = (*T)(t) overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) @@ -837,19 +1366,19 @@ type SetBucketAccessControlPolicyResponse struct { } type SetBucketLoggingStatus struct { - Bucket string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Bucket"` - AWSAccessKeyId string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AWSAccessKeyId,omitempty"` - Timestamp time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` - Signature string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Signature,omitempty"` - Credential string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Credential,omitempty"` - BucketLoggingStatus BucketLoggingStatus `xml:"http://s3.amazonaws.com/doc/2006-03-01/ BucketLoggingStatus"` + Bucket string `xml:"Bucket"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` + BucketLoggingStatus BucketLoggingStatus `xml:"BucketLoggingStatus"` } func (t *SetBucketLoggingStatus) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type T SetBucketLoggingStatus var layout struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } layout.T = (*T)(t) layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) @@ -859,7 +1388,7 @@ func (t *SetBucketLoggingStatus) UnmarshalXML(d *xml.Decoder, start xml.StartEle type T SetBucketLoggingStatus var overlay struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } overlay.T = (*T)(t) overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) @@ -870,20 +1399,20 @@ type SetBucketLoggingStatusResponse struct { } type SetObjectAccessControlPolicy struct { - Bucket string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Bucket"` - Key string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Key"` - AccessControlList AccessControlList `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlList"` - AWSAccessKeyId string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AWSAccessKeyId,omitempty"` - Timestamp time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` - Signature string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Signature,omitempty"` - Credential string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Credential,omitempty"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + AccessControlList AccessControlList `xml:"AccessControlList"` + AWSAccessKeyId string `xml:"AWSAccessKeyId,omitempty"` + Timestamp time.Time `xml:"Timestamp,omitempty"` + Signature string `xml:"Signature,omitempty"` + Credential string `xml:"Credential,omitempty"` } func (t *SetObjectAccessControlPolicy) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type T SetObjectAccessControlPolicy var layout struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } layout.T = (*T)(t) layout.Timestamp = (*xsdDateTime)(&layout.T.Timestamp) @@ -893,7 +1422,7 @@ func (t *SetObjectAccessControlPolicy) UnmarshalXML(d *xml.Decoder, start xml.St type T SetObjectAccessControlPolicy var overlay struct { *T - Timestamp *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Timestamp,omitempty"` + Timestamp *xsdDateTime `xml:"Timestamp,omitempty"` } overlay.T = (*T)(t) overlay.Timestamp = (*xsdDateTime)(&overlay.T.Timestamp) @@ -904,37 +1433,37 @@ type SetObjectAccessControlPolicyResponse struct { } type Status struct { - Code int `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Code"` - Description string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Description"` + Code int `xml:"Code"` + Description string `xml:"Description"` } // May be one of STANDARD, REDUCED_REDUNDANCY, GLACIER, UNKNOWN type StorageClass string type TopicConfiguration struct { - Topic string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Topic"` - Event []string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Event"` + Topic string `xml:"Topic"` + Event []string `xml:"Event"` } type User struct { } type VersionEntry struct { - Key string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Key"` - VersionId string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ VersionId"` - IsLatest bool `xml:"http://s3.amazonaws.com/doc/2006-03-01/ IsLatest"` - LastModified time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"` - ETag string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ETag"` - Size int64 `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Size"` - Owner CanonicalUser `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Owner,omitempty"` - StorageClass StorageClass `xml:"http://s3.amazonaws.com/doc/2006-03-01/ StorageClass"` + Key string `xml:"Key"` + VersionId string `xml:"VersionId"` + IsLatest bool `xml:"IsLatest"` + LastModified time.Time `xml:"LastModified"` + ETag string `xml:"ETag"` + Size int64 `xml:"Size"` + Owner CanonicalUser `xml:"Owner,omitempty"` + StorageClass StorageClass `xml:"StorageClass"` } func (t *VersionEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error { type T VersionEntry var layout struct { *T - LastModified *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"` + LastModified *xsdDateTime `xml:"LastModified"` } layout.T = (*T)(t) layout.LastModified = (*xsdDateTime)(&layout.T.LastModified) @@ -944,7 +1473,7 @@ func (t *VersionEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElement) erro type T VersionEntry var overlay struct { *T - LastModified *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"` + LastModified *xsdDateTime `xml:"LastModified"` } overlay.T = (*T)(t) overlay.LastModified = (*xsdDateTime)(&overlay.T.LastModified) @@ -952,8 +1481,8 @@ func (t *VersionEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElement) erro } type VersioningConfiguration struct { - Status VersioningStatus `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Status,omitempty"` - MfaDelete MfaDeleteStatus `xml:"http://s3.amazonaws.com/doc/2006-03-01/ MfaDelete,omitempty"` + Status VersioningStatus `xml:"Status,omitempty"` + MfaDelete MfaDeleteStatus `xml:"MfaDelete,omitempty"` } // May be one of Enabled, Suspended @@ -976,10 +1505,10 @@ func (b xsdBase64Binary) MarshalText() ([]byte, error) { type xsdDateTime time.Time func (t *xsdDateTime) UnmarshalText(text []byte) error { - return _unmarshalTime(text, (*time.Time)(t), s3TimeFormat) + return _unmarshalTime(text, (*time.Time)(t), "2006-01-02T15:04:05.999999999") } func (t xsdDateTime) MarshalText() ([]byte, error) { - return []byte((time.Time)(t).Format(s3TimeFormat)), nil + return _marshalTime((time.Time)(t), "2006-01-02T15:04:05.999999999") } func (t xsdDateTime) MarshalXML(e *xml.Encoder, start xml.StartElement) error { if (time.Time)(t).IsZero() { @@ -1006,3 +1535,6 @@ func _unmarshalTime(text []byte, t *time.Time, format string) (err error) { } return err } +func _marshalTime(t time.Time, format string) ([]byte, error) { + return []byte(t.Format(format + "Z07:00")), nil +} diff --git a/weed/s3api/s3api_xsd_generated_helper.go b/weed/s3api/s3api_xsd_generated_helper.go new file mode 100644 index 000000000..24cdd2289 --- /dev/null +++ b/weed/s3api/s3api_xsd_generated_helper.go @@ -0,0 +1,10 @@ +package s3api + +type Grantee struct { + XMLNS string `xml:"xmlns:xsi,attr"` + XMLXSI string `xml:"xsi:type,attr"` + Type string `xml:"Type"` + ID string `xml:"ID,omitempty"` + DisplayName string `xml:"DisplayName,omitempty"` + URI string `xml:"URI,omitempty"` +} diff --git a/weed/s3api/s3err/error_handler.go b/weed/s3api/s3err/error_handler.go index 3fb04a313..466c0e61b 100644 --- a/weed/s3api/s3err/error_handler.go +++ b/weed/s3api/s3err/error_handler.go @@ -79,6 +79,7 @@ func setCommonHeaders(w http.ResponseWriter, r *http.Request) { w.Header().Set("Accept-Ranges", "bytes") if r.Header.Get("Origin") != "" { w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Expose-Headers", "*") w.Header().Set("Access-Control-Allow-Credentials", "true") } } diff --git a/weed/security/tls.go b/weed/security/tls.go index ae6510219..1a9dfacb5 100644 --- a/weed/security/tls.go +++ b/weed/security/tls.go @@ -4,16 +4,17 @@ import ( "crypto/tls" "crypto/x509" "fmt" - "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/credentials/tls/certprovider/pemfile" - "google.golang.org/grpc/security/advancedtls" "os" + "slices" "strings" "time" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/util" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials/tls/certprovider/pemfile" + "google.golang.org/grpc/security/advancedtls" ) const CredRefreshingInterval = time.Duration(5) * time.Hour @@ -54,7 +55,7 @@ func LoadServerTLS(config *util.ViperProxy, component string) (grpc.ServerOption } // Start a server and create a client using advancedtls API with Provider. - options := &advancedtls.ServerOptions{ + options := &advancedtls.Options{ IdentityOptions: advancedtls.IdentityCertificateOptions{ IdentityProvider: serverIdentityProvider, }, @@ -62,7 +63,22 @@ func LoadServerTLS(config *util.ViperProxy, component string) (grpc.ServerOption RootProvider: serverRootProvider, }, RequireClientCert: true, - VType: advancedtls.CertVerification, + VerificationType: advancedtls.CertVerification, + } + options.MinTLSVersion, err = TlsVersionByName(config.GetString("tls.min_version")) + if err != nil { + glog.Warningf("tls min version parse failed, %v", err) + return nil, nil + } + options.MaxTLSVersion, err = TlsVersionByName(config.GetString("tls.max_version")) + if err != nil { + glog.Warningf("tls max version parse failed, %v", err) + return nil, nil + } + options.CipherSuites, err = TlsCipherSuiteByNames(config.GetString("tls.cipher_suites")) + if err != nil { + glog.Warningf("tls cipher suite parse failed, %v", err) + return nil, nil } allowedCommonNames := config.GetString(component + ".allowed_commonNames") allowedWildcardDomain := config.GetString("grpc.allowed_wildcard_domain") @@ -75,10 +91,10 @@ func LoadServerTLS(config *util.ViperProxy, component string) (grpc.ServerOption AllowedCommonNames: allowedCommonNamesMap, AllowedWildcardDomain: allowedWildcardDomain, } - options.VerifyPeer = auther.Authenticate + options.AdditionalPeerVerification = auther.Authenticate } else { - options.VerifyPeer = func(params *advancedtls.VerificationFuncParams) (*advancedtls.VerificationResults, error) { - return &advancedtls.VerificationResults{}, nil + options.AdditionalPeerVerification = func(params *advancedtls.HandshakeVerificationInfo) (*advancedtls.PostHandshakeVerificationResults, error) { + return &advancedtls.PostHandshakeVerificationResults{}, nil } } ta, err := advancedtls.NewServerCreds(options) @@ -118,17 +134,17 @@ func LoadClientTLS(config *util.ViperProxy, component string) grpc.DialOption { glog.Warningf("pemfile.NewProvider(%v) failed: %v", clientRootOptions, err) return grpc.WithTransportCredentials(insecure.NewCredentials()) } - options := &advancedtls.ClientOptions{ + options := &advancedtls.Options{ IdentityOptions: advancedtls.IdentityCertificateOptions{ IdentityProvider: clientProvider, }, - VerifyPeer: func(params *advancedtls.VerificationFuncParams) (*advancedtls.VerificationResults, error) { - return &advancedtls.VerificationResults{}, nil + AdditionalPeerVerification: func(params *advancedtls.HandshakeVerificationInfo) (*advancedtls.PostHandshakeVerificationResults, error) { + return &advancedtls.PostHandshakeVerificationResults{}, nil }, RootOptions: advancedtls.RootCertificateOptions{ RootProvider: clientRootProvider, }, - VType: advancedtls.CertVerification, + VerificationType: advancedtls.CertVerification, } ta, err := advancedtls.NewClientCreds(options) if err != nil { @@ -155,14 +171,68 @@ func LoadClientTLSHTTP(clientCertFile string) *tls.Config { } } -func (a Authenticator) Authenticate(params *advancedtls.VerificationFuncParams) (*advancedtls.VerificationResults, error) { +func (a Authenticator) Authenticate(params *advancedtls.HandshakeVerificationInfo) (*advancedtls.PostHandshakeVerificationResults, error) { if a.AllowedWildcardDomain != "" && strings.HasSuffix(params.Leaf.Subject.CommonName, a.AllowedWildcardDomain) { - return &advancedtls.VerificationResults{}, nil + return &advancedtls.PostHandshakeVerificationResults{}, nil } if _, ok := a.AllowedCommonNames[params.Leaf.Subject.CommonName]; ok { - return &advancedtls.VerificationResults{}, nil + return &advancedtls.PostHandshakeVerificationResults{}, nil } err := fmt.Errorf("Authenticate: invalid subject client common name: %s", params.Leaf.Subject.CommonName) glog.Error(err) return nil, err } + +func FixTlsConfig(viper *util.ViperProxy, config *tls.Config) error { + var err error + config.MinVersion, err = TlsVersionByName(viper.GetString("tls.min_version")) + if err != nil { + return err + } + config.MaxVersion, err = TlsVersionByName(viper.GetString("tls.max_version")) + if err != nil { + return err + } + config.CipherSuites, err = TlsCipherSuiteByNames(viper.GetString("tls.cipher_suites")) + return err +} + +func TlsVersionByName(name string) (uint16, error) { + switch name { + case "": + return 0, nil + case "SSLv3": + return tls.VersionSSL30, nil + case "TLS 1.0": + return tls.VersionTLS10, nil + case "TLS 1.1": + return tls.VersionTLS11, nil + case "TLS 1.2": + return tls.VersionTLS12, nil + case "TLS 1.3": + return tls.VersionTLS13, nil + default: + return 0, fmt.Errorf("invalid tls version %s", name) + } +} + +func TlsCipherSuiteByNames(cipherSuiteNames string) ([]uint16, error) { + cipherSuiteNames = strings.TrimSpace(cipherSuiteNames) + if cipherSuiteNames == "" { + return nil, nil + } + names := strings.Split(cipherSuiteNames, ",") + cipherSuites := tls.CipherSuites() + cipherIds := make([]uint16, 0, len(names)) + for _, name := range names { + name = strings.TrimSpace(name) + index := slices.IndexFunc(cipherSuites, func(suite *tls.CipherSuite) bool { + return name == suite.Name + }) + if index == -1 { + return nil, fmt.Errorf("invalid tls cipher suite name %s", name) + } + cipherIds = append(cipherIds, cipherSuites[index].ID) + } + return cipherIds, nil +} diff --git a/weed/sequence/memory_sequencer.go b/weed/sequence/memory_sequencer.go index 6e879bd79..92944266c 100644 --- a/weed/sequence/memory_sequencer.go +++ b/weed/sequence/memory_sequencer.go @@ -30,7 +30,3 @@ func (m *MemorySequencer) SetMax(seenValue uint64) { m.counter = seenValue + 1 } } - -func (m *MemorySequencer) Peek() uint64 { - return m.counter -} diff --git a/weed/sequence/sequence.go b/weed/sequence/sequence.go index 2258d001b..49de63b73 100644 --- a/weed/sequence/sequence.go +++ b/weed/sequence/sequence.go @@ -3,5 +3,4 @@ package sequence type Sequencer interface { NextFileId(count uint64) uint64 SetMax(uint64) - Peek() uint64 } diff --git a/weed/sequence/snowflake_sequencer.go b/weed/sequence/snowflake_sequencer.go index b63198349..05694f681 100644 --- a/weed/sequence/snowflake_sequencer.go +++ b/weed/sequence/snowflake_sequencer.go @@ -42,8 +42,3 @@ func (m *SnowflakeSequencer) NextFileId(count uint64) uint64 { // ignore setmax as we are snowflake func (m *SnowflakeSequencer) SetMax(seenValue uint64) { } - -// return a new id as no Peek is stored -func (m *SnowflakeSequencer) Peek() uint64 { - return uint64(m.node.Generate().Int64()) -} diff --git a/weed/server/common.go b/weed/server/common.go index a7d67fb2e..e6f6cdb88 100644 --- a/weed/server/common.go +++ b/weed/server/common.go @@ -127,7 +127,7 @@ func debug(params ...interface{}) { func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterFn operation.GetMasterFn, grpcDialOption grpc.DialOption) { m := make(map[string]interface{}) - if r.Method != "POST" { + if r.Method != http.MethodPost { writeJsonError(w, r, http.StatusMethodNotAllowed, errors.New("Only submit via POST!")) return } @@ -181,7 +181,12 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterFn ope PairMap: pu.PairMap, Jwt: assignResult.Auth, } - uploadResult, err := operation.UploadData(pu.Data, uploadOption) + uploader, err := operation.NewUploader() + if err != nil { + writeJsonError(w, r, http.StatusInternalServerError, err) + return + } + uploadResult, err := uploader.UploadData(pu.Data, uploadOption) if err != nil { writeJsonError(w, r, http.StatusInternalServerError, err) return @@ -260,7 +265,7 @@ func handleStaticResources2(r *mux.Router) { r.PathPrefix("/seaweedfsstatic/").Handler(http.StripPrefix("/seaweedfsstatic", http.FileServer(http.FS(StaticFS)))) } -func adjustPassthroughHeaders(w http.ResponseWriter, r *http.Request, filename string) { +func AdjustPassthroughHeaders(w http.ResponseWriter, r *http.Request, filename string) { for header, values := range r.Header { if normalizedHeader, ok := s3_constants.PassThroughHeaders[strings.ToLower(header)]; ok { w.Header()[normalizedHeader] = values @@ -284,7 +289,7 @@ func adjustHeaderContentDisposition(w http.ResponseWriter, r *http.Request, file } } -func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, prepareWriteFn func(offset int64, size int64) (filer.DoStreamContent, error)) error { +func ProcessRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, prepareWriteFn func(offset int64, size int64) (filer.DoStreamContent, error)) error { rangeReq := r.Header.Get("Range") bufferedWriter := writePool.Get().(*bufio.Writer) bufferedWriter.Reset(w) @@ -297,14 +302,14 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64 w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10)) writeFn, err := prepareWriteFn(0, totalSize) if err != nil { - glog.Errorf("processRangeRequest: %v", err) + glog.Errorf("ProcessRangeRequest: %v", err) http.Error(w, err.Error(), http.StatusInternalServerError) - return fmt.Errorf("processRangeRequest: %v", err) + return fmt.Errorf("ProcessRangeRequest: %v", err) } if err = writeFn(bufferedWriter); err != nil { - glog.Errorf("processRangeRequest: %v", err) + glog.Errorf("ProcessRangeRequest: %v", err) http.Error(w, err.Error(), http.StatusInternalServerError) - return fmt.Errorf("processRangeRequest: %v", err) + return fmt.Errorf("ProcessRangeRequest: %v", err) } return nil } @@ -313,9 +318,9 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64 //mostly copy from src/pkg/net/http/fs.go ranges, err := parseRange(rangeReq, totalSize) if err != nil { - glog.Errorf("processRangeRequest headers: %+v err: %v", w.Header(), err) + glog.Errorf("ProcessRangeRequest headers: %+v err: %v", w.Header(), err) http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable) - return fmt.Errorf("processRangeRequest header: %v", err) + return fmt.Errorf("ProcessRangeRequest header: %v", err) } if sumRangesSize(ranges) > totalSize { // The total number of bytes in all the ranges @@ -345,16 +350,16 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64 writeFn, err := prepareWriteFn(ra.start, ra.length) if err != nil { - glog.Errorf("processRangeRequest range[0]: %+v err: %v", w.Header(), err) + glog.Errorf("ProcessRangeRequest range[0]: %+v err: %v", w.Header(), err) http.Error(w, err.Error(), http.StatusInternalServerError) - return fmt.Errorf("processRangeRequest: %v", err) + return fmt.Errorf("ProcessRangeRequest: %v", err) } w.WriteHeader(http.StatusPartialContent) err = writeFn(bufferedWriter) if err != nil { - glog.Errorf("processRangeRequest range[0]: %+v err: %v", w.Header(), err) + glog.Errorf("ProcessRangeRequest range[0]: %+v err: %v", w.Header(), err) http.Error(w, err.Error(), http.StatusInternalServerError) - return fmt.Errorf("processRangeRequest range[0]: %v", err) + return fmt.Errorf("ProcessRangeRequest range[0]: %v", err) } return nil } @@ -369,9 +374,9 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64 } writeFn, err := prepareWriteFn(ra.start, ra.length) if err != nil { - glog.Errorf("processRangeRequest range[%d] err: %v", i, err) + glog.Errorf("ProcessRangeRequest range[%d] err: %v", i, err) http.Error(w, "Internal Error", http.StatusInternalServerError) - return fmt.Errorf("processRangeRequest range[%d] err: %v", i, err) + return fmt.Errorf("ProcessRangeRequest range[%d] err: %v", i, err) } writeFnByRange[i] = writeFn } @@ -406,9 +411,9 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64 } w.WriteHeader(http.StatusPartialContent) if _, err := io.CopyN(bufferedWriter, sendContent, sendSize); err != nil { - glog.Errorf("processRangeRequest err: %v", err) + glog.Errorf("ProcessRangeRequest err: %v", err) http.Error(w, "Internal Error", http.StatusInternalServerError) - return fmt.Errorf("processRangeRequest err: %v", err) + return fmt.Errorf("ProcessRangeRequest err: %v", err) } return nil } diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index b9571710d..b1440c94f 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -291,7 +291,7 @@ func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntr glog.V(4).Infof("DeleteEntry %v", req) - err = fs.filer.DeleteEntryMetaAndData(ctx, util.JoinPath(req.Directory, req.Name), req.IsRecursive, req.IgnoreRecursiveError, req.IsDeleteData, req.IsFromOtherCluster, req.Signatures) + err = fs.filer.DeleteEntryMetaAndData(ctx, util.JoinPath(req.Directory, req.Name), req.IsRecursive, req.IgnoreRecursiveError, req.IsDeleteData, req.IsFromOtherCluster, req.Signatures, req.IfNotModifiedAfter) resp = &filer_pb.DeleteEntryResponse{} if err != nil && err != filer_pb.ErrNotFound { resp.Error = err.Error() @@ -363,12 +363,7 @@ func (fs *FilerServer) DeleteCollection(ctx context.Context, req *filer_pb.Delet glog.V(4).Infof("DeleteCollection %v", req) - err = fs.filer.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error { - _, err := client.CollectionDelete(context.Background(), &master_pb.CollectionDeleteRequest{ - Name: req.GetCollection(), - }) - return err - }) + err = fs.filer.DoDeleteCollection(req.GetCollection()) return &filer_pb.DeleteCollectionResponse{}, err } diff --git a/weed/server/filer_grpc_server_admin.go b/weed/server/filer_grpc_server_admin.go index b4caaf4e2..8b4912258 100644 --- a/weed/server/filer_grpc_server_admin.go +++ b/weed/server/filer_grpc_server_admin.go @@ -96,6 +96,8 @@ func (fs *FilerServer) GetFilerConfiguration(ctx context.Context, req *filer_pb. MetricsIntervalSec: int32(fs.metricsIntervalSec), Version: util.Version(), FilerGroup: fs.option.FilerGroup, + MajorVersion: util.MAJOR_VERSION, + MinorVersion: util.MINOR_VERSION, } glog.V(4).Infof("GetFilerConfiguration: %v", t) diff --git a/weed/server/filer_grpc_server_rename.go b/weed/server/filer_grpc_server_rename.go index 3acea6f14..db00dd496 100644 --- a/weed/server/filer_grpc_server_rename.go +++ b/weed/server/filer_grpc_server_rename.go @@ -203,7 +203,7 @@ func (fs *FilerServer) moveSelfEntry(ctx context.Context, stream filer_pb.Seawee // delete old entry ctx = context.WithValue(ctx, "OP", "MV") - deleteErr := fs.filer.DeleteEntryMetaAndData(ctx, oldPath, false, false, false, false, signatures) + deleteErr := fs.filer.DeleteEntryMetaAndData(ctx, oldPath, false, false, false, false, signatures, 0) if deleteErr != nil { return deleteErr } diff --git a/weed/server/filer_grpc_server_traverse_meta.go b/weed/server/filer_grpc_server_traverse_meta.go new file mode 100644 index 000000000..4a924f065 --- /dev/null +++ b/weed/server/filer_grpc_server_traverse_meta.go @@ -0,0 +1,84 @@ +package weed_server + +import ( + "context" + "fmt" + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/util" + "github.com/viant/ptrie" +) + +func (fs *FilerServer) TraverseBfsMetadata(req *filer_pb.TraverseBfsMetadataRequest, stream filer_pb.SeaweedFiler_TraverseBfsMetadataServer) error { + + glog.V(0).Infof("TraverseBfsMetadata %v", req) + + excludedTrie := ptrie.New[bool]() + for _, excluded := range req.ExcludedPrefixes { + excludedTrie.Put([]byte(excluded), true) + } + + ctx := stream.Context() + + queue := util.NewQueue[*filer.Entry]() + dirEntry, err := fs.filer.FindEntry(ctx, util.FullPath(req.Directory)) + if err != nil { + return fmt.Errorf("find dir %s: %v", req.Directory, err) + } + queue.Enqueue(dirEntry) + + for item := queue.Dequeue(); item != nil; item = queue.Dequeue() { + if excludedTrie.MatchPrefix([]byte(item.FullPath), func(key []byte, value bool) bool { + return true + }) { + // println("excluded", item.FullPath) + continue + } + parent, _ := item.FullPath.DirAndName() + if err := stream.Send(&filer_pb.TraverseBfsMetadataResponse{ + Directory: parent, + Entry: item.ToProtoEntry(), + }); err != nil { + return fmt.Errorf("send traverse bfs metadata response: %v", err) + } + + if !item.IsDirectory() { + continue + } + + if err := fs.iterateDirectory(ctx, item.FullPath, func(entry *filer.Entry) error { + queue.Enqueue(entry) + return nil + }); err != nil { + return err + } + } + + return nil +} + +func (fs *FilerServer) iterateDirectory(ctx context.Context, dirPath util.FullPath, fn func(entry *filer.Entry) error) (err error) { + var lastFileName string + var listErr error + for { + var hasEntries bool + lastFileName, listErr = fs.filer.StreamListDirectoryEntries(ctx, dirPath, lastFileName, false, 1024, "", "", "", func(entry *filer.Entry) bool { + hasEntries = true + if fnErr := fn(entry); fnErr != nil { + err = fnErr + return false + } + return true + }) + if listErr != nil { + return listErr + } + if err != nil { + return err + } + if !hasEntries { + return nil + } + } +} diff --git a/weed/server/filer_grpc_server_traverse_meta_test.go b/weed/server/filer_grpc_server_traverse_meta_test.go new file mode 100644 index 000000000..72f8a916e --- /dev/null +++ b/weed/server/filer_grpc_server_traverse_meta_test.go @@ -0,0 +1,31 @@ +package weed_server + +import ( + "github.com/stretchr/testify/assert" + "github.com/viant/ptrie" + "testing" +) + +func TestPtrie(t *testing.T) { + b := []byte("/topics/abc/dev") + excludedTrie := ptrie.New[bool]() + excludedTrie.Put([]byte("/topics/abc/d"), true) + excludedTrie.Put([]byte("/topics/abc"), true) + + assert.True(t, excludedTrie.MatchPrefix(b, func(key []byte, value bool) bool { + println("matched1", string(key)) + return true + })) + + assert.True(t, excludedTrie.MatchAll(b, func(key []byte, value bool) bool { + println("matched2", string(key)) + return true + })) + + assert.False(t, excludedTrie.MatchAll([]byte("/topics/ab"), func(key []byte, value bool) bool { + println("matched3", string(key)) + return true + })) + + assert.False(t, excludedTrie.Has(b)) +} diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 0b7254c0d..ee052579c 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -74,7 +74,6 @@ type FilerOption struct { DiskType string AllowedOrigins []string ExposeDirectoryData bool - JoinExistingFiler bool } type FilerServer struct { @@ -198,12 +197,9 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) existingNodes := fs.filer.ListExistingPeerUpdates(context.Background()) startFromTime := time.Now().Add(-filer.LogFlushInterval) - if option.JoinExistingFiler { - startFromTime = time.Time{} - } if isFresh { glog.V(0).Infof("%s bootstrap from peers %+v", option.Host, existingNodes) - if err := fs.filer.MaybeBootstrapFromPeers(option.Host, existingNodes, startFromTime); err != nil { + if err := fs.filer.MaybeBootstrapFromOnePeer(option.Host, existingNodes, startFromTime); err != nil { glog.Fatalf("%s bootstrap from %+v: %v", option.Host, existingNodes, err) } } diff --git a/weed/server/filer_server_handlers_proxy.go b/weed/server/filer_server_handlers_proxy.go index e04994569..c1a26ca11 100644 --- a/weed/server/filer_server_handlers_proxy.go +++ b/weed/server/filer_server_handlers_proxy.go @@ -3,24 +3,13 @@ package weed_server import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/security" - "github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util/mem" "io" "math/rand" "net/http" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) -var ( - client *http.Client -) - -func init() { - client = &http.Client{Transport: &http.Transport{ - MaxIdleConns: 1024, - MaxIdleConnsPerHost: 1024, - }} -} - func (fs *FilerServer) maybeAddVolumeJwtAuthorization(r *http.Request, fileId string, isWrite bool) { encodedJwt := fs.maybeGetVolumeJwtAuthorizationToken(fileId, isWrite) @@ -71,14 +60,14 @@ func (fs *FilerServer) proxyToVolumeServer(w http.ResponseWriter, r *http.Reques } } - proxyResponse, postErr := client.Do(proxyReq) + proxyResponse, postErr := util_http.GetGlobalHttpClient().Do(proxyReq) if postErr != nil { glog.Errorf("post to filer: %v", postErr) w.WriteHeader(http.StatusInternalServerError) return } - defer util.CloseResponse(proxyResponse) + defer util_http.CloseResponse(proxyResponse) for k, v := range proxyResponse.Header { w.Header()[k] = v diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 123b7a494..a02e6c2c1 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -71,14 +71,14 @@ func checkPreconditions(w http.ResponseWriter, r *http.Request, entry *filer.Ent ifModifiedSinceHeader := r.Header.Get("If-Modified-Since") if ifNoneMatchETagHeader != "" { if util.CanonicalizeETag(etag) == util.CanonicalizeETag(ifNoneMatchETagHeader) { - setEtag(w, etag) + SetEtag(w, etag) w.WriteHeader(http.StatusNotModified) return true } } else if ifModifiedSinceHeader != "" { if t, parseError := time.Parse(http.TimeFormat, ifModifiedSinceHeader); parseError == nil { if !t.Before(entry.Attr.Mtime) { - setEtag(w, etag) + SetEtag(w, etag) w.WriteHeader(http.StatusNotModified) return true } @@ -220,13 +220,13 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) w.Header().Set(s3_constants.AmzTagCount, strconv.Itoa(tagCount)) } - setEtag(w, etag) + SetEtag(w, etag) filename := entry.Name() - adjustPassthroughHeaders(w, r, filename) + AdjustPassthroughHeaders(w, r, filename) totalSize := int64(entry.Size()) - if r.Method == "HEAD" { + if r.Method == http.MethodHead { w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10)) return } @@ -252,7 +252,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) } } - processRangeRequest(r, w, totalSize, mimeType, func(offset int64, size int64) (filer.DoStreamContent, error) { + ProcessRangeRequest(r, w, totalSize, mimeType, func(offset int64, size int64) (filer.DoStreamContent, error) { if offset+size <= int64(len(entry.Content)) { return func(writer io.Writer) error { _, err := writer.Write(entry.Content[offset : offset+size]) diff --git a/weed/server/filer_server_handlers_read_dir.go b/weed/server/filer_server_handlers_read_dir.go index 0531527bb..56f0f9cb4 100644 --- a/weed/server/filer_server_handlers_read_dir.go +++ b/weed/server/filer_server_handlers_read_dir.go @@ -31,8 +31,8 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque path = path[:len(path)-1] } - limit, limit_err := strconv.Atoi(r.FormValue("limit")) - if limit_err != nil { + limit, limitErr := strconv.Atoi(r.FormValue("limit")) + if limitErr != nil { limit = fs.option.DirListingLimit } @@ -62,6 +62,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque if r.Header.Get("Accept") == "application/json" { writeJsonQuiet(w, r, http.StatusOK, struct { + Version string Path string Entries interface{} Limit int @@ -69,6 +70,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque ShouldDisplayLoadMore bool EmptyFolder bool }{ + util.Version(), path, entries, limit, @@ -80,6 +82,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque } err = ui.StatusTpl.Execute(w, struct { + Version string Path string Breadcrumbs []ui.Breadcrumb Entries interface{} @@ -89,6 +92,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque EmptyFolder bool ShowDirectoryDelete bool }{ + util.Version(), path, ui.ToBreadcrumb(path), entries, diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index b186fd34e..f0f756e34 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -18,6 +18,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/util" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var ( @@ -120,7 +121,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request, conte fs.autoChunk(ctx, w, r, contentLength, so) } - util.CloseRequest(r) + util_http.CloseRequest(r) } @@ -211,7 +212,7 @@ func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { objectPath = objectPath[0 : len(objectPath)-1] } - err := fs.filer.DeleteEntryMetaAndData(context.Background(), util.FullPath(objectPath), isRecursive, ignoreRecursiveError, !skipChunkDeletion, false, nil) + err := fs.filer.DeleteEntryMetaAndData(context.Background(), util.FullPath(objectPath), isRecursive, ignoreRecursiveError, !skipChunkDeletion, false, nil, 0) if err != nil { if err == filer_pb.ErrNotFound { writeJsonQuiet(w, r, http.StatusNoContent, nil) diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index 2698e2209..1c7ed0c3c 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -39,7 +39,7 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r * var reply *FilerPostResult var err error var md5bytes []byte - if r.Method == "POST" { + if r.Method == http.MethodPost { if r.Header.Get("Content-Type") == "" && strings.HasSuffix(r.URL.Path, "/") { reply, err = fs.mkdir(ctx, w, r, so) } else { @@ -148,6 +148,10 @@ func skipCheckParentDirEntry(r *http.Request) bool { return r.URL.Query().Get("skipCheckParentDir") == "true" } +func isS3Request(r *http.Request) bool { + return r.Header.Get(s3_constants.AmzAuthType) != "" || r.Header.Get("X-Amz-Date") != "" +} + func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileName string, contentType string, so *operation.StorageOption, md5bytes []byte, fileChunks []*filer_pb.FileChunk, chunkOffset int64, content []byte) (filerResult *FilerPostResult, replyerr error) { // detect file mode @@ -266,7 +270,12 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa } } - if dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil, skipCheckParentDirEntry(r), so.MaxFileNameLength); dbErr != nil { + dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil, skipCheckParentDirEntry(r), so.MaxFileNameLength) + // In test_bucket_listv2_delimiter_basic, the valid object key is the parent folder + if dbErr != nil && strings.HasSuffix(dbErr.Error(), " is a file") && isS3Request(r) { + dbErr = fs.filer.CreateEntry(ctx, entry, false, false, nil, true, so.MaxFileNameLength) + } + if dbErr != nil { replyerr = dbErr filerResult.Error = dbErr.Error() glog.V(0).Infof("failing to write %s to filer server : %v", path, dbErr) @@ -299,8 +308,14 @@ func (fs *FilerServer) saveAsChunk(so *operation.StorageOption) filer.SaveDataAs PairMap: nil, Jwt: auth, } + + uploader, uploaderErr := operation.NewUploader() + if uploaderErr != nil { + return uploaderErr + } + var uploadErr error - uploadResult, uploadErr, _ = operation.Upload(reader, uploadOption) + uploadResult, uploadErr, _ = uploader.Upload(reader, uploadOption) if uploadErr != nil { return uploadErr } diff --git a/weed/server/filer_server_handlers_write_cipher.go b/weed/server/filer_server_handlers_write_cipher.go index 6cf7d65b1..f8d129bf3 100644 --- a/weed/server/filer_server_handlers_write_cipher.go +++ b/weed/server/filer_server_handlers_write_cipher.go @@ -53,7 +53,13 @@ func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *ht PairMap: pu.PairMap, Jwt: auth, } - uploadResult, uploadError := operation.UploadData(uncompressedData, uploadOption) + + uploader, uploaderErr := operation.NewUploader() + if uploaderErr != nil { + return nil, fmt.Errorf("uploader initialization error: %v", uploaderErr) + } + + uploadResult, uploadError := uploader.UploadData(uncompressedData, uploadOption) if uploadError != nil { return nil, fmt.Errorf("upload to volume server: %v", uploadError) } diff --git a/weed/server/filer_server_handlers_write_upload.go b/weed/server/filer_server_handlers_write_upload.go index 8c8eba078..d0d1575cf 100644 --- a/weed/server/filer_server_handlers_write_upload.go +++ b/weed/server/filer_server_handlers_write_upload.go @@ -158,7 +158,13 @@ func (fs *FilerServer) doUpload(urlLocation string, limitedReader io.Reader, fil PairMap: pairMap, Jwt: auth, } - uploadResult, err, data := operation.Upload(limitedReader, uploadOption) + + uploader, err := operation.NewUploader() + if err != nil { + return nil, err, []byte{} + } + + uploadResult, err, data := uploader.Upload(limitedReader, uploadOption) if uploadResult != nil && uploadResult.RetryCount > 0 { stats.FilerHandlerCounter.WithLabelValues(stats.ChunkUploadRetry).Add(float64(uploadResult.RetryCount)) } diff --git a/weed/server/filer_ui/breadcrumb.go b/weed/server/filer_ui/breadcrumb.go index abb6cce9a..638638196 100644 --- a/weed/server/filer_ui/breadcrumb.go +++ b/weed/server/filer_ui/breadcrumb.go @@ -13,6 +13,9 @@ type Breadcrumb struct { func ToBreadcrumb(fullpath string) (crumbs []Breadcrumb) { parts := strings.Split(fullpath, "/") + if fullpath == "/" { + parts = []string{""} + } for i := 0; i < len(parts); i++ { name := parts[i] diff --git a/weed/server/filer_ui/breadcrumb_test.go b/weed/server/filer_ui/breadcrumb_test.go new file mode 100644 index 000000000..6e42541cb --- /dev/null +++ b/weed/server/filer_ui/breadcrumb_test.go @@ -0,0 +1,86 @@ +package filer_ui + +import ( + "reflect" + "testing" +) + +func TestToBreadcrumb(t *testing.T) { + type args struct { + fullpath string + } + tests := []struct { + name string + args args + wantCrumbs []Breadcrumb + }{ + { + name: "empty", + args: args{ + fullpath: "", + }, + wantCrumbs: []Breadcrumb{ + { + Name: "/", + Link: "/", + }, + }, + }, + { + name: "test1", + args: args{ + fullpath: "/", + }, + wantCrumbs: []Breadcrumb{ + { + Name: "/", + Link: "/", + }, + }, + }, + { + name: "test2", + args: args{ + fullpath: "/abc", + }, + wantCrumbs: []Breadcrumb{ + { + Name: "/", + Link: "/", + }, + { + Name: "abc", + Link: "/abc/", + }, + }, + }, + { + name: "test3", + args: args{ + fullpath: "/abc/def", + }, + wantCrumbs: []Breadcrumb{ + { + Name: "/", + Link: "/", + }, + { + Name: "abc", + Link: "/abc/", + }, + { + Name: "def", + Link: "/abc/def/", + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if gotCrumbs := ToBreadcrumb(tt.args.fullpath); !reflect.DeepEqual(gotCrumbs, tt.wantCrumbs) { + t.Errorf("ToBreadcrumb() = %v, want %v", gotCrumbs, tt.wantCrumbs) + } + }) + } +} diff --git a/weed/server/filer_ui/filer.html b/weed/server/filer_ui/filer.html index 28425f180..627f3ba77 100644 --- a/weed/server/filer_ui/filer.html +++ b/weed/server/filer_ui/filer.html @@ -1,7 +1,7 @@ <!DOCTYPE html> <html> <head> - <title>SeaweedFS Filer</title> + <title>SeaweedFS Filer {{ .Version }}</title> <meta name="viewport" content="width=device-width, initial-scale=1"> <link rel="stylesheet" href="/seaweedfsstatic/bootstrap/3.3.1/css/bootstrap.min.css"> <style> @@ -82,7 +82,7 @@ <div class="page-header"> <h1> <a href="https://github.com/seaweedfs/seaweedfs"><img src="/seaweedfsstatic/seaweed50x50.png"></img></a> - SeaweedFS Filer + SeaweedFS Filer <small>{{ .Version }}</small> </h1> </div> <div class="row"> @@ -99,7 +99,7 @@ {{ range $entry := .Breadcrumbs }} <li><a href="{{ printpath $entry.Link }}"> {{ $entry.Name }} - </li></a> + </a></li> {{ end }} </ol> </div> diff --git a/weed/server/master_grpc_server_assign.go b/weed/server/master_grpc_server_assign.go index 7455c9ea4..4f95b4ff6 100644 --- a/weed/server/master_grpc_server_assign.go +++ b/weed/server/master_grpc_server_assign.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/stats" "time" "github.com/seaweedfs/raft" @@ -69,7 +70,12 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest MemoryMapMaxSizeMb: req.MemoryMapMaxSizeMb, } + if !ms.Topo.DataCenterExists(option.DataCenter) { + return nil, fmt.Errorf("data center %v not found in topology", option.DataCenter) + } + vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl, option.DiskType) + vl.SetLastGrowCount(req.WritableVolumeCount) var ( lastErr error @@ -80,18 +86,17 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest for time.Now().Sub(startTime) < maxTimeout { fid, count, dnList, shouldGrow, err := ms.Topo.PickForWrite(req.Count, option, vl) if shouldGrow && !vl.HasGrowRequest() { - // if picked volume is almost full, trigger a volume-grow request - if ms.Topo.AvailableSpaceFor(option) <= 0 { - return nil, fmt.Errorf("no free volumes left for " + option.String()) + if err != nil && ms.Topo.AvailableSpaceFor(option) <= 0 { + err = fmt.Errorf("%s and no free volumes left for %s", err.Error(), option.String()) } vl.AddGrowRequest() ms.volumeGrowthRequestChan <- &topology.VolumeGrowRequest{ Option: option, - Count: int(req.WritableVolumeCount), + Count: req.WritableVolumeCount, } } if err != nil { - // glog.Warningf("PickForWrite %+v: %v", req, err) + stats.MasterPickForWriteErrorCounter.Inc() lastErr = err time.Sleep(200 * time.Millisecond) continue diff --git a/weed/server/master_grpc_server_volume.go b/weed/server/master_grpc_server_volume.go index 503da7fd4..3cad627db 100644 --- a/weed/server/master_grpc_server_volume.go +++ b/weed/server/master_grpc_server_volume.go @@ -3,11 +3,14 @@ package weed_server import ( "context" "fmt" + "math/rand" "reflect" "strings" "sync" "time" + "github.com/seaweedfs/seaweedfs/weed/topology" + "github.com/seaweedfs/raft" "github.com/seaweedfs/seaweedfs/weed/glog" @@ -18,8 +21,39 @@ import ( "github.com/seaweedfs/seaweedfs/weed/storage/types" ) +func (ms *MasterServer) DoAutomaticVolumeGrow(req *topology.VolumeGrowRequest) { + glog.V(1).Infoln("starting automatic volume grow") + start := time.Now() + newVidLocations, err := ms.vg.AutomaticGrowByType(req.Option, ms.grpcDialOption, ms.Topo, req.Count) + glog.V(1).Infoln("finished automatic volume grow, cost ", time.Now().Sub(start)) + if err != nil { + glog.V(1).Infof("automatic volume grow failed: %+v", err) + return + } + for _, newVidLocation := range newVidLocations { + ms.broadcastToClients(&master_pb.KeepConnectedResponse{VolumeLocation: newVidLocation}) + } +} + func (ms *MasterServer) ProcessGrowRequest() { go func() { + for { + time.Sleep(14*time.Minute + time.Duration(120*rand.Float32())*time.Second) + if !ms.Topo.IsLeader() { + continue + } + for _, vl := range ms.Topo.ListVolumeLyauts() { + if !vl.HasGrowRequest() && vl.ShouldGrowVolumes(&topology.VolumeGrowOption{}) { + vl.AddGrowRequest() + ms.volumeGrowthRequestChan <- &topology.VolumeGrowRequest{ + Option: vl.ToGrowOption(), + Count: vl.GetLastGrowCount(), + } + } + } + } + }() + go func() { filter := sync.Map{} for { req, ok := <-ms.volumeGrowthRequestChan @@ -27,9 +61,13 @@ func (ms *MasterServer) ProcessGrowRequest() { break } + option := req.Option + vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl, option.DiskType) + if !ms.Topo.IsLeader() { //discard buffered requests time.Sleep(time.Second * 1) + vl.DoneGrowRequest() continue } @@ -42,28 +80,15 @@ func (ms *MasterServer) ProcessGrowRequest() { return !found }) - option := req.Option - vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl, option.DiskType) - // not atomic but it's okay if !found && vl.ShouldGrowVolumes(option) { filter.Store(req, nil) // we have lock called inside vg - go func() { - glog.V(1).Infoln("starting automatic volume grow") - start := time.Now() - newVidLocations, err := ms.vg.AutomaticGrowByType(req.Option, ms.grpcDialOption, ms.Topo, req.Count) - glog.V(1).Infoln("finished automatic volume grow, cost ", time.Now().Sub(start)) - if err == nil { - for _, newVidLocation := range newVidLocations { - ms.broadcastToClients(&master_pb.KeepConnectedResponse{VolumeLocation: newVidLocation}) - } - } + go func(req *topology.VolumeGrowRequest, vl *topology.VolumeLayout) { + ms.DoAutomaticVolumeGrow(req) vl.DoneGrowRequest() - filter.Delete(req) - }() - + }(req, vl) } else { glog.V(4).Infoln("discard volume grow request") time.Sleep(time.Millisecond * 211) @@ -91,6 +116,7 @@ func (ms *MasterServer) LookupVolume(ctx context.Context, req *master_pb.LookupV Url: loc.Url, PublicUrl: loc.PublicUrl, DataCenter: loc.DataCenter, + GrpcPort: uint32(loc.GrpcPort), }) } var auth string diff --git a/weed/server/master_server.go b/weed/server/master_server.go index 3499a2e13..44a1664c0 100644 --- a/weed/server/master_server.go +++ b/weed/server/master_server.go @@ -30,6 +30,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/topology" "github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/wdclient" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) const ( @@ -92,15 +93,15 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se v.SetDefault("master.replication.treat_replication_as_minimums", false) replicationAsMin := v.GetBool("master.replication.treat_replication_as_minimums") - v.SetDefault("master.volume_growth.copy_1", 7) - v.SetDefault("master.volume_growth.copy_2", 6) - v.SetDefault("master.volume_growth.copy_3", 3) - v.SetDefault("master.volume_growth.copy_other", 1) - v.SetDefault("master.volume_growth.threshold", 0.9) - topology.VolumeGrowStrategy.Copy1Count = v.GetInt("master.volume_growth.copy_1") - topology.VolumeGrowStrategy.Copy2Count = v.GetInt("master.volume_growth.copy_2") - topology.VolumeGrowStrategy.Copy3Count = v.GetInt("master.volume_growth.copy_3") - topology.VolumeGrowStrategy.CopyOtherCount = v.GetInt("master.volume_growth.copy_other") + v.SetDefault("master.volume_growth.copy_1", topology.VolumeGrowStrategy.Copy1Count) + v.SetDefault("master.volume_growth.copy_2", topology.VolumeGrowStrategy.Copy2Count) + v.SetDefault("master.volume_growth.copy_3", topology.VolumeGrowStrategy.Copy3Count) + v.SetDefault("master.volume_growth.copy_other", topology.VolumeGrowStrategy.CopyOtherCount) + v.SetDefault("master.volume_growth.threshold", topology.VolumeGrowStrategy.Threshold) + topology.VolumeGrowStrategy.Copy1Count = v.GetUint32("master.volume_growth.copy_1") + topology.VolumeGrowStrategy.Copy2Count = v.GetUint32("master.volume_growth.copy_2") + topology.VolumeGrowStrategy.Copy3Count = v.GetUint32("master.volume_growth.copy_3") + topology.VolumeGrowStrategy.CopyOtherCount = v.GetUint32("master.volume_growth.copy_other") topology.VolumeGrowStrategy.Threshold = v.GetFloat64("master.volume_growth.threshold") var preallocateSize int64 @@ -185,22 +186,7 @@ func (ms *MasterServer) SetRaftServer(raftServer *RaftServer) { raftServerName = fmt.Sprintf("[%s]", ms.Topo.RaftServer.Name()) } else if raftServer.RaftHashicorp != nil { ms.Topo.HashicorpRaft = raftServer.RaftHashicorp - leaderCh := raftServer.RaftHashicorp.LeaderCh() - prevLeader, _ := ms.Topo.HashicorpRaft.LeaderWithID() raftServerName = ms.Topo.HashicorpRaft.String() - go func() { - for { - select { - case isLeader := <-leaderCh: - ms.Topo.RaftServerAccessLock.RLock() - leader, _ := ms.Topo.HashicorpRaft.LeaderWithID() - ms.Topo.RaftServerAccessLock.RUnlock() - glog.V(0).Infof("is leader %+v change event: %+v => %+v", isLeader, prevLeader, leader) - stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", leader)).Inc() - prevLeader = leader - } - } - }() } ms.Topo.RaftServerAccessLock.Unlock() @@ -256,7 +242,7 @@ func (ms *MasterServer) proxyToLeader(f http.HandlerFunc) http.HandlerFunc { } director(req) } - proxy.Transport = util.Transport + proxy.Transport = util_http.GetGlobalHttpClient().GetClientTransport() proxy.ServeHTTP(w, r) } } diff --git a/weed/server/master_server_handlers.go b/weed/server/master_server_handlers.go index e4188420d..5e17bcca8 100644 --- a/weed/server/master_server_handlers.go +++ b/weed/server/master_server_handlers.go @@ -2,12 +2,13 @@ package weed_server import ( "fmt" - "github.com/seaweedfs/seaweedfs/weed/glog" "net/http" "strconv" "strings" "time" + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/stats" @@ -74,7 +75,10 @@ func (ms *MasterServer) findVolumeLocation(collection, vid string) operation.Loo machines := ms.Topo.Lookup(collection, volumeId) for _, loc := range machines { locations = append(locations, operation.Location{ - Url: loc.Url(), PublicUrl: loc.PublicUrl, DataCenter: loc.GetDataCenterId(), + Url: loc.Url(), + PublicUrl: loc.PublicUrl, + DataCenter: loc.GetDataCenterId(), + GrpcPort: loc.GrpcPort, }) } } @@ -82,7 +86,10 @@ func (ms *MasterServer) findVolumeLocation(collection, vid string) operation.Loo machines, getVidLocationsErr := ms.MasterClient.GetVidLocations(vid) for _, loc := range machines { locations = append(locations, operation.Location{ - Url: loc.Url, PublicUrl: loc.PublicUrl, DataCenter: loc.DataCenter, + Url: loc.Url, + PublicUrl: loc.PublicUrl, + DataCenter: loc.DataCenter, + GrpcPort: loc.GrpcPort, }) } err = getVidLocationsErr @@ -107,7 +114,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) requestedCount = 1 } - writableVolumeCount, e := strconv.Atoi(r.FormValue("writableVolumeCount")) + writableVolumeCount, e := strconv.ParseUint(r.FormValue("writableVolumeCount"), 10, 32) if e != nil { writableVolumeCount = 0 } @@ -126,23 +133,28 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) startTime = time.Now() ) + if !ms.Topo.DataCenterExists(option.DataCenter) { + writeJsonQuiet(w, r, http.StatusBadRequest, operation.AssignResult{ + Error: fmt.Sprintf("data center %v not found in topology", option.DataCenter), + }) + return + } + for time.Now().Sub(startTime) < maxTimeout { fid, count, dnList, shouldGrow, err := ms.Topo.PickForWrite(requestedCount, option, vl) if shouldGrow && !vl.HasGrowRequest() { - // if picked volume is almost full, trigger a volume-grow request glog.V(0).Infof("dirAssign volume growth %v from %v", option.String(), r.RemoteAddr) - if ms.Topo.AvailableSpaceFor(option) <= 0 { - writeJsonQuiet(w, r, http.StatusNotFound, operation.AssignResult{Error: "No free volumes left for " + option.String()}) - return + if err != nil && ms.Topo.AvailableSpaceFor(option) <= 0 { + err = fmt.Errorf("%s and no free volumes left for %s", err.Error(), option.String()) } vl.AddGrowRequest() ms.volumeGrowthRequestChan <- &topology.VolumeGrowRequest{ Option: option, - Count: writableVolumeCount, + Count: uint32(writableVolumeCount), } } if err != nil { - // glog.Warningf("PickForWrite %+v: %v", req, err) + stats.MasterPickForWriteErrorCounter.Inc() lastErr = err time.Sleep(200 * time.Millisecond) continue diff --git a/weed/server/master_server_handlers_admin.go b/weed/server/master_server_handlers_admin.go index f40b819af..7479b5535 100644 --- a/weed/server/master_server_handlers_admin.go +++ b/weed/server/master_server_handlers_admin.go @@ -18,6 +18,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/storage/types" "github.com/seaweedfs/seaweedfs/weed/topology" "github.com/seaweedfs/seaweedfs/weed/util" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.Request) { @@ -70,7 +71,7 @@ func (ms *MasterServer) volumeVacuumHandler(w http.ResponseWriter, r *http.Reque } func (ms *MasterServer) volumeGrowHandler(w http.ResponseWriter, r *http.Request) { - count := 0 + count := uint64(0) option, err := ms.getVolumeGrowOption(r) if err != nil { writeJsonError(w, r, http.StatusNotAcceptable, err) @@ -78,13 +79,16 @@ func (ms *MasterServer) volumeGrowHandler(w http.ResponseWriter, r *http.Request } glog.V(0).Infof("volumeGrowHandler received %v from %v", option.String(), r.RemoteAddr) - if count, err = strconv.Atoi(r.FormValue("count")); err == nil { - if ms.Topo.AvailableSpaceFor(option) < int64(count*option.ReplicaPlacement.GetCopyCount()) { - err = fmt.Errorf("only %d volumes left, not enough for %d", ms.Topo.AvailableSpaceFor(option), count*option.ReplicaPlacement.GetCopyCount()) + if count, err = strconv.ParseUint(r.FormValue("count"), 10, 32); err == nil { + replicaCount := int64(count * uint64(option.ReplicaPlacement.GetCopyCount())) + if ms.Topo.AvailableSpaceFor(option) < replicaCount { + err = fmt.Errorf("only %d volumes left, not enough for %d", ms.Topo.AvailableSpaceFor(option), replicaCount) + } else if !ms.Topo.DataCenterExists(option.DataCenter) { + err = fmt.Errorf("data center %v not found in topology", option.DataCenter) } else { var newVidLocations []*master_pb.VolumeLocation - newVidLocations, err = ms.vg.GrowByCountAndType(ms.grpcDialOption, count, option, ms.Topo) - count = len(newVidLocations) + newVidLocations, err = ms.vg.GrowByCountAndType(ms.grpcDialOption, uint32(count), option, ms.Topo) + count = uint64(len(newVidLocations)) } } else { err = fmt.Errorf("can not parse parameter count %s", r.FormValue("count")) @@ -110,11 +114,11 @@ func (ms *MasterServer) redirectHandler(w http.ResponseWriter, r *http.Request) location := ms.findVolumeLocation(collection, vid) if location.Error == "" { loc := location.Locations[rand.Intn(len(location.Locations))] - var url string + url, _ := util_http.NormalizeUrl(loc.PublicUrl) if r.URL.RawQuery != "" { - url = util.NormalizeUrl(loc.PublicUrl) + r.URL.Path + "?" + r.URL.RawQuery + url = url + r.URL.Path + "?" + r.URL.RawQuery } else { - url = util.NormalizeUrl(loc.PublicUrl) + r.URL.Path + url = url + r.URL.Path } http.Redirect(w, r, url, http.StatusPermanentRedirect) } else { diff --git a/weed/server/raft_hashicorp.go b/weed/server/raft_hashicorp.go index d06066b93..299df323a 100644 --- a/weed/server/raft_hashicorp.go +++ b/weed/server/raft_hashicorp.go @@ -5,6 +5,14 @@ package weed_server import ( "fmt" + "math/rand" + "os" + "path" + "path/filepath" + "sort" + "strings" + "time" + transport "github.com/Jille/raft-grpc-transport" "github.com/armon/go-metrics" "github.com/armon/go-metrics/prometheus" @@ -14,13 +22,6 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/stats" "google.golang.org/grpc" - "math/rand" - "os" - "path" - "path/filepath" - "sort" - "strings" - "time" ) const ( @@ -56,46 +57,61 @@ func (s *RaftServer) AddPeersConfiguration() (cfg raft.Configuration) { return cfg } -func (s *RaftServer) UpdatePeers() { +func (s *RaftServer) monitorLeaderLoop(updatePeers bool) { for { + prevLeader, _ := s.RaftHashicorp.LeaderWithID() select { case isLeader := <-s.RaftHashicorp.LeaderCh(): + leader, _ := s.RaftHashicorp.LeaderWithID() if isLeader { - peerLeader := string(s.serverAddr) - existsPeerName := make(map[string]bool) - for _, server := range s.RaftHashicorp.GetConfiguration().Configuration().Servers { - if string(server.ID) == peerLeader { - continue - } - existsPeerName[string(server.ID)] = true - } - for _, peer := range s.peers { - peerName := string(peer) - if peerName == peerLeader || existsPeerName[peerName] { - continue - } - glog.V(0).Infof("adding new peer: %s", peerName) - s.RaftHashicorp.AddVoter( - raft.ServerID(peerName), raft.ServerAddress(peer.ToGrpcAddress()), 0, 0) - } - for peer := range existsPeerName { - if _, found := s.peers[peer]; !found { - glog.V(0).Infof("removing old peer: %s", peer) - s.RaftHashicorp.RemoveServer(raft.ServerID(peer), 0, 0) - } - } - if _, found := s.peers[peerLeader]; !found { - glog.V(0).Infof("removing old leader peer: %s", peerLeader) - s.RaftHashicorp.RemoveServer(raft.ServerID(peerLeader), 0, 0) + + if updatePeers { + s.updatePeers() + updatePeers = false } + + s.topo.DoBarrier() + + stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", leader)).Inc() + } else { + s.topo.BarrierReset() } - return - case <-time.After(updatePeersTimeout): - return + glog.V(0).Infof("is leader %+v change event: %+v => %+v", isLeader, prevLeader, leader) + prevLeader = leader } } } +func (s *RaftServer) updatePeers() { + peerLeader := string(s.serverAddr) + existsPeerName := make(map[string]bool) + for _, server := range s.RaftHashicorp.GetConfiguration().Configuration().Servers { + if string(server.ID) == peerLeader { + continue + } + existsPeerName[string(server.ID)] = true + } + for _, peer := range s.peers { + peerName := string(peer) + if peerName == peerLeader || existsPeerName[peerName] { + continue + } + glog.V(0).Infof("adding new peer: %s", peerName) + s.RaftHashicorp.AddVoter( + raft.ServerID(peerName), raft.ServerAddress(peer.ToGrpcAddress()), 0, 0) + } + for peer := range existsPeerName { + if _, found := s.peers[peer]; !found { + glog.V(0).Infof("removing old peer: %s", peer) + s.RaftHashicorp.RemoveServer(raft.ServerID(peer), 0, 0) + } + } + if _, found := s.peers[peerLeader]; !found { + glog.V(0).Infof("removing old leader peer: %s", peerLeader) + s.RaftHashicorp.RemoveServer(raft.ServerID(peerLeader), 0, 0) + } +} + func NewHashicorpRaftServer(option *RaftServerOption) (*RaftServer, error) { s := &RaftServer{ peers: option.Peers, @@ -157,6 +173,8 @@ func NewHashicorpRaftServer(option *RaftServerOption) (*RaftServer, error) { if err != nil { return nil, fmt.Errorf("raft.NewRaft: %v", err) } + + updatePeers := false if option.RaftBootstrap || len(s.RaftHashicorp.GetConfiguration().Configuration().Servers) == 0 { cfg := s.AddPeersConfiguration() // Need to get lock, in case all servers do this at the same time. @@ -169,9 +187,11 @@ func NewHashicorpRaftServer(option *RaftServerOption) (*RaftServer, error) { return nil, fmt.Errorf("raft.Raft.BootstrapCluster: %v", err) } } else { - go s.UpdatePeers() + updatePeers = true } + go s.monitorLeaderLoop(updatePeers) + ticker := time.NewTicker(c.HeartbeatTimeout * 10) if glog.V(4) { go func() { diff --git a/weed/server/raft_server.go b/weed/server/raft_server.go index d718ecac7..4bcd808c2 100644 --- a/weed/server/raft_server.go +++ b/weed/server/raft_server.go @@ -2,13 +2,14 @@ package weed_server import ( "encoding/json" - transport "github.com/Jille/raft-grpc-transport" "io" "math/rand" "os" "path" "time" + transport "github.com/Jille/raft-grpc-transport" + "google.golang.org/grpc" "github.com/seaweedfs/seaweedfs/weed/pb" diff --git a/weed/server/volume_grpc_remote.go b/weed/server/volume_grpc_remote.go index 64254b3b8..4452e019b 100644 --- a/weed/server/volume_grpc_remote.go +++ b/weed/server/volume_grpc_remote.go @@ -70,10 +70,15 @@ func (vs *VolumeServer) FetchAndWriteNeedle(ctx context.Context, req *volume_ser PairMap: nil, Jwt: security.EncodedJwt(req.Auth), } - if _, replicaWriteErr := operation.UploadData(data, uploadOption); replicaWriteErr != nil { - if err == nil { - err = fmt.Errorf("remote write needle %d size %d: %v", req.NeedleId, req.Size, err) - } + + uploader, uploaderErr := operation.NewUploader() + if uploaderErr != nil && err == nil { + err = fmt.Errorf("remote write needle %d size %d: %v", req.NeedleId, req.Size, uploaderErr) + return + } + + if _, replicaWriteErr := uploader.UploadData(data, uploadOption); replicaWriteErr != nil && err == nil { + err = fmt.Errorf("remote write needle %d size %d: %v", req.NeedleId, req.Size, replicaWriteErr) } }(replica.Url) } diff --git a/weed/server/volume_server_handlers_read.go b/weed/server/volume_server_handlers_read.go index cc364513b..15d639f49 100644 --- a/weed/server/volume_server_handlers_read.go +++ b/weed/server/volume_server_handlers_read.go @@ -27,6 +27,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/storage" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/util" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`) @@ -81,10 +82,11 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) } if vs.ReadMode == "proxy" { // proxy client request to target server - u, _ := url.Parse(util.NormalizeUrl(lookupResult.Locations[0].Url)) + rawURL, _ := util_http.NormalizeUrl(lookupResult.Locations[0].Url) + u, _ := url.Parse(rawURL) r.URL.Host = u.Host r.URL.Scheme = u.Scheme - request, err := http.NewRequest("GET", r.URL.String(), nil) + request, err := http.NewRequest(http.MethodGet, r.URL.String(), nil) if err != nil { glog.V(0).Infof("failed to instance http request of url %s: %v", r.URL.String(), err) InternalError(w) @@ -96,13 +98,13 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) } } - response, err := client.Do(request) + response, err := util_http.GetGlobalHttpClient().Do(request) if err != nil { glog.V(0).Infof("request remote url %s: %v", r.URL.String(), err) InternalError(w) return } - defer util.CloseResponse(response) + defer util_http.CloseResponse(response) // proxy target response to client for k, vv := range response.Header { for _, v := range vv { @@ -116,7 +118,8 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) return } else { // redirect - u, _ := url.Parse(util.NormalizeUrl(lookupResult.Locations[0].PublicUrl)) + rawURL, _ := util_http.NormalizeUrl(lookupResult.Locations[0].PublicUrl) + u, _ := url.Parse(rawURL) u.Path = fmt.Sprintf("%s/%s,%s", u.Path, vid, fid) arg := url.Values{} if c := r.FormValue("collection"); c != "" { @@ -186,7 +189,7 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) w.WriteHeader(http.StatusNotModified) return } - setEtag(w, n.Etag()) + SetEtag(w, n.Etag()) if n.HasPairs() { pairMap := make(map[string]string) @@ -253,7 +256,7 @@ func shouldAttemptStreamWrite(hasLocalVolume bool, ext string, r *http.Request) if len(ext) > 0 { ext = strings.ToLower(ext) } - if r.Method == "HEAD" { + if r.Method == http.MethodHead { return true, true } _, _, _, shouldResize := shouldResizeImages(ext, r) @@ -377,14 +380,14 @@ func writeResponseContent(filename, mimeType string, rs io.ReadSeeker, w http.Re } w.Header().Set("Accept-Ranges", "bytes") - adjustPassthroughHeaders(w, r, filename) + AdjustPassthroughHeaders(w, r, filename) - if r.Method == "HEAD" { + if r.Method == http.MethodHead { w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10)) return nil } - return processRangeRequest(r, w, totalSize, mimeType, func(offset int64, size int64) (filer.DoStreamContent, error) { + return ProcessRangeRequest(r, w, totalSize, mimeType, func(offset int64, size int64) (filer.DoStreamContent, error) { return func(writer io.Writer) error { if _, e = rs.Seek(offset, 0); e != nil { return e @@ -406,14 +409,14 @@ func (vs *VolumeServer) streamWriteResponseContent(filename string, mimeType str w.Header().Set("Content-Type", mimeType) } w.Header().Set("Accept-Ranges", "bytes") - adjustPassthroughHeaders(w, r, filename) + AdjustPassthroughHeaders(w, r, filename) - if r.Method == "HEAD" { + if r.Method == http.MethodHead { w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10)) return } - processRangeRequest(r, w, totalSize, mimeType, func(offset int64, size int64) (filer.DoStreamContent, error) { + ProcessRangeRequest(r, w, totalSize, mimeType, func(offset int64, size int64) (filer.DoStreamContent, error) { return func(writer io.Writer) error { return vs.store.ReadVolumeNeedleDataInto(volumeId, n, readOption, writer, offset, size) }, nil diff --git a/weed/server/volume_server_handlers_write.go b/weed/server/volume_server_handlers_write.go index 6e151bf80..7f0fcc871 100644 --- a/weed/server/volume_server_handlers_write.go +++ b/weed/server/volume_server_handlers_write.go @@ -53,7 +53,7 @@ func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) { // http 204 status code does not allow body if writeError == nil && isUnchanged { - setEtag(w, reqNeedle.Etag()) + SetEtag(w, reqNeedle.Etag()) w.WriteHeader(http.StatusNoContent) return } @@ -65,7 +65,7 @@ func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) { ret.Size = uint32(originalSize) ret.ETag = reqNeedle.Etag() ret.Mime = string(reqNeedle.Mime) - setEtag(w, ret.ETag) + SetEtag(w, ret.ETag) w.Header().Set("Content-MD5", contentMd5) writeJsonQuiet(w, r, httpStatus, ret) } @@ -147,7 +147,7 @@ func writeDeleteResult(err error, count int64, w http.ResponseWriter, r *http.Re } } -func setEtag(w http.ResponseWriter, etag string) { +func SetEtag(w http.ResponseWriter, etag string) { if etag != "" { if strings.HasPrefix(etag, "\"") { w.Header().Set("ETag", etag) diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go index 97d51dad7..dbe6dfed5 100644 --- a/weed/server/webdav_server.go +++ b/weed/server/webdav_server.go @@ -99,6 +99,7 @@ type FileInfo struct { modifiedTime time.Time etag string isDirectory bool + err error } func (fi *FileInfo) Name() string { return fi.name } @@ -109,6 +110,9 @@ func (fi *FileInfo) IsDir() bool { return fi.isDirectory } func (fi *FileInfo) Sys() interface{} { return nil } func (fi *FileInfo) ETag(ctx context.Context) (string, error) { + if fi.err != nil { + return "", fi.err + } return fi.etag, nil } @@ -269,7 +273,10 @@ func (fs *WebDavFileSystem) OpenFile(ctx context.Context, fullFilePath string, f fi, err := fs.stat(ctx, fullFilePath) if err != nil { - return nil, os.ErrNotExist + if err == os.ErrNotExist { + return nil, err + } + return &WebDavFile{fs: fs}, nil } if !strings.HasSuffix(fullFilePath, "/") && fi.IsDir() { fullFilePath += "/" @@ -365,12 +372,16 @@ func (fs *WebDavFileSystem) stat(ctx context.Context, fullFilePath string) (os.F var fi FileInfo entry, err := filer_pb.GetEntry(fs, fullpath) + if err != nil { + if err == filer_pb.ErrNotFound { + return nil, os.ErrNotExist + } + fi.err = err + return &fi, nil + } if entry == nil { return nil, os.ErrNotExist } - if err != nil { - return nil, err - } fi.size = int64(filer.FileSize(entry)) fi.name = string(fullpath) fi.mode = os.FileMode(entry.Attributes.FileMode) @@ -392,8 +403,13 @@ func (fs *WebDavFileSystem) Stat(ctx context.Context, name string) (os.FileInfo, } func (f *WebDavFile) saveDataAsChunk(reader io.Reader, name string, offset int64, tsNs int64) (chunk *filer_pb.FileChunk, err error) { + uploader, uploaderErr := operation.NewUploader() + if uploaderErr != nil { + glog.V(0).Infof("upload data %v: %v", f.name, uploaderErr) + return nil, fmt.Errorf("upload data: %v", uploaderErr) + } - fileId, uploadResult, flushErr, _ := operation.UploadWithRetry( + fileId, uploadResult, flushErr, _ := uploader.UploadWithRetry( f.fs, &filer_pb.AssignVolumeRequest{ Count: 1, @@ -509,7 +525,9 @@ func (f *WebDavFile) Write(buf []byte) (int, error) { func (f *WebDavFile) Close() error { glog.V(2).Infof("WebDavFileSystem.Close %v", f.name) - + if f.bufWriter == nil { + return nil + } err := f.bufWriter.Close() if f.entry != nil { diff --git a/weed/shell/command_ec_balance.go b/weed/shell/command_ec_balance.go index 17ba63cfe..217e5750e 100644 --- a/weed/shell/command_ec_balance.go +++ b/weed/shell/command_ec_balance.go @@ -455,16 +455,20 @@ func doBalanceEcRack(commandEnv *CommandEnv, ecRack *EcRack, applyBalancing bool func pickOneEcNodeAndMoveOneShard(commandEnv *CommandEnv, averageShardsPerEcNode int, existingLocation *EcNode, collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, possibleDestinationEcNodes []*EcNode, applyBalancing bool) error { sortEcNodesByFreeslotsDescending(possibleDestinationEcNodes) - + skipReason := "" for _, destEcNode := range possibleDestinationEcNodes { + if destEcNode.info.Id == existingLocation.info.Id { continue } if destEcNode.freeEcSlot <= 0 { + skipReason += fmt.Sprintf(" Skipping %s because it has no free slots\n", destEcNode.info.Id) continue } if findEcVolumeShards(destEcNode, vid).ShardIdCount() >= averageShardsPerEcNode { + skipReason += fmt.Sprintf(" Skipping %s because it %d >= avernageShards (%d)\n", + destEcNode.info.Id, findEcVolumeShards(destEcNode, vid).ShardIdCount(), averageShardsPerEcNode) continue } @@ -477,7 +481,7 @@ func pickOneEcNodeAndMoveOneShard(commandEnv *CommandEnv, averageShardsPerEcNode return nil } - + fmt.Printf("WARNING: Could not find suitable taget node for %d.%d:\n%s", vid, shardId, skipReason) return nil } diff --git a/weed/shell/command_ec_rebuild.go b/weed/shell/command_ec_rebuild.go index 2131c5649..a4dfac67c 100644 --- a/weed/shell/command_ec_rebuild.go +++ b/weed/shell/command_ec_rebuild.go @@ -224,7 +224,7 @@ func prepareDataToRecover(commandEnv *CommandEnv, rebuilder *EcNode, collection Collection: collection, ShardIds: []uint32{uint32(shardId)}, CopyEcxFile: needEcxFile, - CopyEcjFile: needEcxFile, + CopyEcjFile: true, CopyVifFile: needEcxFile, SourceDataNode: ecNodes[0].info.Id, }) diff --git a/weed/shell/command_fs_merge_volumes.go b/weed/shell/command_fs_merge_volumes.go index 4a6048a43..b77feb8e3 100644 --- a/weed/shell/command_fs_merge_volumes.go +++ b/weed/shell/command_fs_merge_volumes.go @@ -19,14 +19,10 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/util" -) - -var ( - client *http.Client + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) func init() { - client = &http.Client{} Commands = append(Commands, &commandFsMergeVolumes{}) } @@ -104,7 +100,7 @@ func (c *commandFsMergeVolumes) Do(args []string, commandEnv *CommandEnv, writer return nil } - defer client.CloseIdleConnections() + defer util_http.GetGlobalHttpClient().CloseIdleConnections() return commandEnv.WithFilerClient(false, func(filerClient filer_pb.SeaweedFilerClient) error { return filer_pb.TraverseBfs(commandEnv, util.FullPath(dir), func(parentPath util.FullPath, entry *filer_pb.Entry) { @@ -304,7 +300,7 @@ func moveChunk(chunk *filer_pb.FileChunk, toVolumeId needle.VolumeId, masterClie if err != nil { return err } - defer util.CloseResponse(resp) + defer util_http.CloseResponse(resp) defer reader.Close() var filename string @@ -322,7 +318,12 @@ func moveChunk(chunk *filer_pb.FileChunk, toVolumeId needle.VolumeId, masterClie isCompressed := resp.Header.Get("Content-Encoding") == "gzip" md5 := resp.Header.Get("Content-MD5") - _, err, _ = operation.Upload(reader, &operation.UploadOption{ + uploader, err := operation.NewUploader() + if err != nil { + return err + } + + _, err, _ = uploader.Upload(reader, &operation.UploadOption{ UploadUrl: uploadURL, Filename: filename, IsInputCompressed: isCompressed, @@ -342,18 +343,18 @@ func moveChunk(chunk *filer_pb.FileChunk, toVolumeId needle.VolumeId, masterClie func readUrl(fileUrl string) (*http.Response, io.ReadCloser, error) { - req, err := http.NewRequest("GET", fileUrl, nil) + req, err := http.NewRequest(http.MethodGet, fileUrl, nil) if err != nil { return nil, nil, err } req.Header.Add("Accept-Encoding", "gzip") - r, err := client.Do(req) + r, err := util_http.GetGlobalHttpClient().Do(req) if err != nil { return nil, nil, err } if r.StatusCode >= 400 { - util.CloseResponse(r) + util_http.CloseResponse(r) return nil, nil, fmt.Errorf("%s: %s", fileUrl, r.Status) } diff --git a/weed/shell/command_fs_verify.go b/weed/shell/command_fs_verify.go index 32d498202..47052cca0 100644 --- a/weed/shell/command_fs_verify.go +++ b/weed/shell/command_fs_verify.go @@ -1,6 +1,7 @@ package shell import ( + "bytes" "context" "flag" "fmt" @@ -30,6 +31,7 @@ type commandFsVerify struct { volumeServers []pb.ServerAddress volumeIds map[uint32][]pb.ServerAddress verbose *bool + metadataFromLog *bool concurrency *int modifyTimeAgoAtSec int64 writer io.Writer @@ -56,7 +58,7 @@ func (c *commandFsVerify) Do(args []string, commandEnv *CommandEnv, writer io.Wr c.verbose = fsVerifyCommand.Bool("v", false, "print out each processed files") modifyTimeAgo := fsVerifyCommand.Duration("modifyTimeAgo", 0, "only include files after this modify time to verify") c.concurrency = fsVerifyCommand.Int("concurrency", 0, "number of parallel verification per volume server") - + c.metadataFromLog = fsVerifyCommand.Bool("metadataFromLog", false, "Using filer log to get metadata") if err = fsVerifyCommand.Parse(args); err != nil { return err } @@ -88,14 +90,19 @@ func (c *commandFsVerify) Do(args []string, commandEnv *CommandEnv, writer io.Wr defer close(c.waitChan[volumeServerStr]) } } - - fCount, eConut, terr := c.verifyTraverseBfs(path) - if terr == nil { - fmt.Fprintf(writer, "verified %d files, error %d files \n", fCount, eConut) + var fCount, eCount uint64 + if *c.metadataFromLog { + var wg sync.WaitGroup + fCount, eCount, err = c.verifyProcessMetadata(path, &wg) + wg.Wait() + if err != nil { + return err + } + } else { + fCount, eCount, err = c.verifyTraverseBfs(path) } - - return terr - + fmt.Fprintf(writer, "verified %d files, error %d files \n", fCount, eCount) + return err } func (c *commandFsVerify) collectVolumeIds() error { @@ -117,7 +124,7 @@ func (c *commandFsVerify) collectVolumeIds() error { return nil } -func (c *commandFsVerify) verifyEntry(volumeServer pb.ServerAddress, fileId *filer_pb.FileId) error { +func (c *commandFsVerify) verifyChunk(volumeServer pb.ServerAddress, fileId *filer_pb.FileId) error { err := operation.WithVolumeServerClient(false, volumeServer, c.env.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error { _, err := client.VolumeNeedleStatus(context.Background(), @@ -138,6 +145,126 @@ type ItemEntry struct { path util.FullPath } +func (c *commandFsVerify) verifyProcessMetadata(path string, wg *sync.WaitGroup) (fileCount uint64, errCount uint64, err error) { + processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error { + message := resp.EventNotification + if resp.EventNotification.NewEntry == nil { + return nil + } + chunkCount := len(message.NewEntry.Chunks) + if chunkCount == 0 { + return nil + } + entryPath := fmt.Sprintf("%s/%s", message.NewParentPath, message.NewEntry.Name) + errorChunksCount := atomic.NewUint64(0) + if !c.verifyEntry(entryPath, message.NewEntry.Chunks, errorChunksCount, wg) { + if err = c.env.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + entryResp, errReq := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{ + Directory: message.NewParentPath, + Name: message.NewEntry.Name, + }) + if errReq != nil { + if strings.HasSuffix(errReq.Error(), "no entry is found in filer store") { + return nil + } + return errReq + } + if entryResp.Entry.Attributes.Mtime == message.NewEntry.Attributes.Mtime && + bytes.Equal(entryResp.Entry.Attributes.Md5, message.NewEntry.Attributes.Md5) { + fmt.Fprintf(c.writer, "file: %s needles:%d failed:%d\n", entryPath, chunkCount, errorChunksCount.Load()) + errCount++ + } + return nil + }); err != nil { + return err + } + return nil + } + if *c.verbose { + fmt.Fprintf(c.writer, "file: %s needles:%d verifed\n", entryPath, chunkCount) + } + fileCount++ + return nil + } + metadataFollowOption := &pb.MetadataFollowOption{ + ClientName: "shell_verify", + ClientId: util.RandomInt32(), + ClientEpoch: 0, + SelfSignature: 0, + PathPrefix: path, + AdditionalPathPrefixes: nil, + DirectoriesToWatch: nil, + StartTsNs: time.Now().Add(-1 * time.Second * time.Duration(c.modifyTimeAgoAtSec)).UnixNano(), + StopTsNs: time.Now().UnixNano(), + EventErrorType: pb.DontLogError, + } + return fileCount, errCount, pb.FollowMetadata(c.env.option.FilerAddress, c.env.option.GrpcDialOption, metadataFollowOption, processEventFn) +} + +func (c *commandFsVerify) verifyEntry(path string, chunks []*filer_pb.FileChunk, errorCount *atomic.Uint64, wg *sync.WaitGroup) bool { + fileMsg := fmt.Sprintf("file:%s", path) + itemIsVerifed := atomic.NewBool(true) + for _, chunk := range chunks { + if volumeIds, ok := c.volumeIds[chunk.Fid.VolumeId]; ok { + for _, volumeServer := range volumeIds { + if *c.concurrency == 0 { + if err := c.verifyChunk(volumeServer, chunk.Fid); err != nil { + if !(*c.metadataFromLog && strings.HasSuffix(err.Error(), "not found")) { + fmt.Fprintf(c.writer, "%s failed verify fileId %s: %+v\n", + fileMsg, chunk.GetFileIdString(), err) + } + if itemIsVerifed.Load() { + itemIsVerifed.Store(false) + errorCount.Add(1) + } + } + continue + } + c.waitChanLock.RLock() + waitChan, ok := c.waitChan[string(volumeServer)] + c.waitChanLock.RUnlock() + if !ok { + fmt.Fprintf(c.writer, "%s failed to get channel for %s fileId: %s\n", + string(volumeServer), fileMsg, chunk.GetFileIdString()) + if itemIsVerifed.Load() { + itemIsVerifed.Store(false) + errorCount.Add(1) + } + continue + } + wg.Add(1) + waitChan <- struct{}{} + go func(fChunk *filer_pb.FileChunk, path string, volumeServer pb.ServerAddress, msg string) { + defer wg.Done() + if err := c.verifyChunk(volumeServer, fChunk.Fid); err != nil { + if !(*c.metadataFromLog && strings.HasSuffix(err.Error(), "not found")) { + fmt.Fprintf(c.writer, "%s failed verify fileId %s: %+v\n", + msg, fChunk.GetFileIdString(), err) + } + if itemIsVerifed.Load() { + itemIsVerifed.Store(false) + errorCount.Add(1) + } + } + <-waitChan + }(chunk, path, volumeServer, fileMsg) + } + } else { + if !*c.metadataFromLog { + err := fmt.Errorf("volumeId %d not found", chunk.Fid.VolumeId) + fmt.Fprintf(c.writer, "%s failed verify fileId %s: %+v\n", + fileMsg, chunk.GetFileIdString(), err) + } + if itemIsVerifed.Load() { + itemIsVerifed.Store(false) + errorCount.Add(1) + } + break + } + } + return itemIsVerifed.Load() +} + func (c *commandFsVerify) verifyTraverseBfs(path string) (fileCount uint64, errCount uint64, err error) { timeNowAtSec := time.Now().Unix() return fileCount, errCount, doTraverseBfsAndSaving(c.env, c.writer, path, false, @@ -166,63 +293,9 @@ func (c *commandFsVerify) verifyTraverseBfs(path string) (fileCount uint64, errC for itemEntry := range outputChan { i := itemEntry.(*ItemEntry) itemPath := string(i.path) - fileMsg := fmt.Sprintf("file:%s", itemPath) - itemIsVerifed := atomic.NewBool(true) - for _, chunk := range i.chunks { - if volumeIds, ok := c.volumeIds[chunk.Fid.VolumeId]; ok { - for _, volumeServer := range volumeIds { - if *c.concurrency == 0 { - if err = c.verifyEntry(volumeServer, chunk.Fid); err != nil { - fmt.Fprintf(c.writer, "%s failed verify fileId %s: %+v\n", - fileMsg, chunk.GetFileIdString(), err) - if itemIsVerifed.Load() { - itemIsVerifed.Store(false) - itemErrCount.Add(1) - } - } - continue - } - c.waitChanLock.RLock() - waitChan, ok := c.waitChan[string(volumeServer)] - c.waitChanLock.RUnlock() - if !ok { - fmt.Fprintf(c.writer, "%s failed to get channel for %s fileId: %s: %+v\n", - string(volumeServer), fileMsg, chunk.GetFileIdString(), err) - if itemIsVerifed.Load() { - itemIsVerifed.Store(false) - itemErrCount.Add(1) - } - continue - } - wg.Add(1) - waitChan <- struct{}{} - go func(fChunk *filer_pb.FileChunk, path string, volumeServer pb.ServerAddress, msg string) { - defer wg.Done() - if err = c.verifyEntry(volumeServer, fChunk.Fid); err != nil { - fmt.Fprintf(c.writer, "%s failed verify fileId %s: %+v\n", - msg, fChunk.GetFileIdString(), err) - if itemIsVerifed.Load() { - itemIsVerifed.Store(false) - itemErrCount.Add(1) - } - } - <-waitChan - }(chunk, itemPath, volumeServer, fileMsg) - } - } else { - err = fmt.Errorf("volumeId %d not found", chunk.Fid.VolumeId) - fmt.Fprintf(c.writer, "%s failed verify fileId %s: %+v\n", - fileMsg, chunk.GetFileIdString(), err) - if itemIsVerifed.Load() { - itemIsVerifed.Store(false) - itemErrCount.Add(1) - } - break - } - } - if itemIsVerifed.Load() { + if c.verifyEntry(itemPath, i.chunks, itemErrCount, &wg) { if *c.verbose { - fmt.Fprintf(c.writer, "%s needles:%d verifed\n", fileMsg, len(i.chunks)) + fmt.Fprintf(c.writer, "file: %s needles:%d verifed\n", itemPath, len(i.chunks)) } fileCount++ } diff --git a/weed/shell/command_remote_uncache.go b/weed/shell/command_remote_uncache.go index 34269ce4e..25e51ff74 100644 --- a/weed/shell/command_remote_uncache.go +++ b/weed/shell/command_remote_uncache.go @@ -7,6 +7,7 @@ import ( "io" "path/filepath" "strings" + "time" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" @@ -164,12 +165,12 @@ func (ff *FileFilter) matches(entry *filer_pb.Entry) bool { } } if *ff.minAge != -1 { - if entry.Attributes.Crtime < *ff.minAge { + if entry.Attributes.Crtime + *ff.minAge > time.Now().Unix() { return false } } if *ff.maxAge != -1 { - if entry.Attributes.Crtime > *ff.maxAge { + if entry.Attributes.Crtime + *ff.maxAge < time.Now().Unix() { return false } } diff --git a/weed/shell/command_s3_bucket_quota_check.go b/weed/shell/command_s3_bucket_quota_check.go index bc0d838f7..b130e4fad 100644 --- a/weed/shell/command_s3_bucket_quota_check.go +++ b/weed/shell/command_s3_bucket_quota_check.go @@ -130,7 +130,7 @@ func (c *commandS3BucketQuotaEnforce) processEachBucket(fc *filer.FilerConf, fil } else { fmt.Fprintf(writer, " changing bucket %s to writable.\n", entry.Name) } - fc.AddLocationConf(locConf) + fc.SetLocationConf(locConf) } return diff --git a/weed/shell/command_s3_clean_uploads.go b/weed/shell/command_s3_clean_uploads.go index 2be61f72a..accce60ba 100644 --- a/weed/shell/command_s3_clean_uploads.go +++ b/weed/shell/command_s3_clean_uploads.go @@ -12,6 +12,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) func init() { @@ -90,7 +91,7 @@ func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io deleteUrl := fmt.Sprintf("http://%s%s/%s?recursive=true&ignoreRecursiveError=true", commandEnv.option.FilerAddress.ToHttpAddress(), uploadsDir, staleUpload) fmt.Fprintf(writer, "purge %s\n", deleteUrl) - err = util.Delete(deleteUrl, string(encodedJwt)) + err = util_http.Delete(deleteUrl, string(encodedJwt)) if err != nil && err.Error() != "" { return fmt.Errorf("purge %s/%s: %v", uploadsDir, staleUpload, err) } diff --git a/weed/shell/command_volume_check_disk.go b/weed/shell/command_volume_check_disk.go index 3e2512bdd..0e76f6ac9 100644 --- a/weed/shell/command_volume_check_disk.go +++ b/weed/shell/command_volume_check_disk.go @@ -279,7 +279,7 @@ func doVolumeCheckDisk(minuend, subtrahend *needle_map.MemDb, source, target *Vo fmt.Fprintf(writer, "delete %s %s => %s\n", needleValue.Key.FileId(source.info.Id), source.location.dataNode.Id, target.location.dataNode.Id) } } - deleteResults, deleteErr := operation.DeleteFilesAtOneVolumeServer( + deleteResults, deleteErr := operation.DeleteFileIdsAtOneVolumeServer( pb.NewServerAddressFromDataNode(target.location.dataNode), grpcDialOption, fidList, false) if deleteErr != nil { diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index 1d27fae1d..acb0ee5ad 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -7,6 +7,18 @@ import ( "errors" "flag" "fmt" + "io" + "math" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/pb" @@ -14,23 +26,12 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" "github.com/seaweedfs/seaweedfs/weed/storage" - "github.com/seaweedfs/seaweedfs/weed/storage/idx" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/needle_map" "github.com/seaweedfs/seaweedfs/weed/storage/types" "github.com/seaweedfs/seaweedfs/weed/util" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" "golang.org/x/sync/errgroup" - "io" - "math" - "net/http" - "net/url" - "os" - "path" - "path/filepath" - "strconv" - "strings" - "sync" - "time" ) func init() { @@ -163,7 +164,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. delete(volumeIdToVInfo, volumeId) continue } - err = c.collectOneVolumeFileIds(dataNodeId, volumeId, vinfo, uint64(collectModifyFromAtNs), uint64(collectCutoffFromAtNs)) + err = c.collectOneVolumeFileIds(dataNodeId, volumeId, vinfo) if err != nil { return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, err) } @@ -198,7 +199,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. return fmt.Errorf("failed to collect file ids from filer: %v", err) } // volume file ids subtract filer file ids - if err = c.findExtraChunksInVolumeServers(dataNodeVolumeIdToVInfo, *applyPurging); err != nil { + if err = c.findExtraChunksInVolumeServers(dataNodeVolumeIdToVInfo, *applyPurging, uint64(collectModifyFromAtNs), uint64(collectCutoffFromAtNs)); err != nil { return fmt.Errorf("findExtraChunksInVolumeServers: %v", err) } } @@ -288,7 +289,7 @@ func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInf return nil } -func (c *commandVolumeFsck) findExtraChunksInVolumeServers(dataNodeVolumeIdToVInfo map[string]map[uint32]VInfo, applyPurging bool) error { +func (c *commandVolumeFsck) findExtraChunksInVolumeServers(dataNodeVolumeIdToVInfo map[string]map[uint32]VInfo, applyPurging bool, modifyFrom, cutoffFrom uint64) error { var totalInUseCount, totalOrphanChunkCount, totalOrphanDataSize uint64 volumeIdOrphanFileIds := make(map[uint32]map[string]bool) @@ -298,7 +299,7 @@ func (c *commandVolumeFsck) findExtraChunksInVolumeServers(dataNodeVolumeIdToVIn serverReplicas := make(map[uint32][]pb.ServerAddress) for dataNodeId, volumeIdToVInfo := range dataNodeVolumeIdToVInfo { for volumeId, vinfo := range volumeIdToVInfo { - inUseCount, orphanFileIds, orphanDataSize, checkErr := c.oneVolumeFileIdsSubtractFilerFileIds(dataNodeId, volumeId, &vinfo) + inUseCount, orphanFileIds, orphanDataSize, checkErr := c.oneVolumeFileIdsSubtractFilerFileIds(dataNodeId, volumeId, &vinfo, modifyFrom, cutoffFrom) if checkErr != nil { return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, checkErr) } @@ -394,7 +395,7 @@ func (c *commandVolumeFsck) findExtraChunksInVolumeServers(dataNodeVolumeIdToVIn return nil } -func (c *commandVolumeFsck) collectOneVolumeFileIds(dataNodeId string, volumeId uint32, vinfo VInfo, modifyFrom uint64, cutoffFrom uint64) error { +func (c *commandVolumeFsck) collectOneVolumeFileIds(dataNodeId string, volumeId uint32, vinfo VInfo) error { if *c.verbose { fmt.Fprintf(c.writer, "collecting volume %d file ids from %s ...\n", volumeId, vinfo.server) @@ -431,29 +432,6 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(dataNodeId string, volumeId } buf.Write(resp.FileContent) } - if !vinfo.isReadOnly && (modifyFrom != 0 || cutoffFrom != 0) { - index, err := idx.FirstInvalidIndex(buf.Bytes(), - func(key types.NeedleId, offset types.Offset, size types.Size) (bool, error) { - resp, err := volumeServerClient.ReadNeedleMeta(context.Background(), &volume_server_pb.ReadNeedleMetaRequest{ - VolumeId: volumeId, - NeedleId: uint64(key), - Offset: offset.ToActualOffset(), - Size: int32(size), - }) - if err != nil { - return false, fmt.Errorf("read needle meta with id %d from volume %d: %v", key, volumeId, err) - } - if (modifyFrom == 0 || modifyFrom <= resp.AppendAtNs) && (cutoffFrom == 0 || resp.AppendAtNs <= cutoffFrom) { - return true, nil - } - return false, nil - }) - if err != nil { - fmt.Fprintf(c.writer, "Failed to search for last valid index on volume %d with error %v\n", volumeId, err) - } else { - buf.Truncate(index * types.NeedleMapEntrySize) - } - } idxFilename := getVolumeFileIdFile(c.tempFolder, dataNodeId, volumeId) err = writeToFile(buf.Bytes(), idxFilename) if err != nil { @@ -552,9 +530,7 @@ func (c *commandVolumeFsck) httpDelete(path util.FullPath) { fmt.Fprintf(c.writer, "HTTP delete request error: %v\n", err) } - client := &http.Client{} - - resp, err := client.Do(req) + resp, err := util_http.GetGlobalHttpClient().Do(req) if err != nil { fmt.Fprintf(c.writer, "DELETE fetch error: %v\n", err) } @@ -571,7 +547,7 @@ func (c *commandVolumeFsck) httpDelete(path util.FullPath) { } } -func (c *commandVolumeFsck) oneVolumeFileIdsSubtractFilerFileIds(dataNodeId string, volumeId uint32, vinfo *VInfo) (inUseCount uint64, orphanFileIds []string, orphanDataSize uint64, err error) { +func (c *commandVolumeFsck) oneVolumeFileIdsSubtractFilerFileIds(dataNodeId string, volumeId uint32, vinfo *VInfo, modifyFrom, cutoffFrom uint64) (inUseCount uint64, orphanFileIds []string, orphanDataSize uint64, err error) { volumeFileIdDb := needle_map.NewMemDb() defer volumeFileIdDb.Close() @@ -611,9 +587,30 @@ func (c *commandVolumeFsck) oneVolumeFileIdsSubtractFilerFileIds(dataNodeId stri if n.Size.IsDeleted() { return nil } - orphanFileIds = append(orphanFileIds, n.Key.FileId(volumeId)) - orphanFileCount++ - orphanDataSize += uint64(n.Size) + if cutoffFrom > 0 || modifyFrom > 0 { + return operation.WithVolumeServerClient(false, vinfo.server, c.env.option.GrpcDialOption, + func(volumeServerClient volume_server_pb.VolumeServerClient) error { + resp, err := volumeServerClient.ReadNeedleMeta(context.Background(), &volume_server_pb.ReadNeedleMetaRequest{ + VolumeId: volumeId, + NeedleId: types.NeedleIdToUint64(n.Key), + Offset: n.Offset.ToActualOffset(), + Size: int32(n.Size), + }) + if err != nil { + return fmt.Errorf("read needle meta with id %d from volume %d: %v", n.Key, volumeId, err) + } + if (modifyFrom == 0 || modifyFrom <= resp.AppendAtNs) && (cutoffFrom == 0 || resp.AppendAtNs <= cutoffFrom) { + orphanFileIds = append(orphanFileIds, n.Key.FileId(volumeId)) + orphanFileCount++ + orphanDataSize += uint64(n.Size) + } + return nil + }) + } else { + orphanFileIds = append(orphanFileIds, n.Key.FileId(volumeId)) + orphanFileCount++ + orphanDataSize += uint64(n.Size) + } return nil }); err != nil { err = fmt.Errorf("failed to AscendingVisit %+v", err) @@ -697,7 +694,7 @@ func (c *commandVolumeFsck) purgeFileIdsForOneVolume(volumeId uint32, fileIds [] go func(server pb.ServerAddress, fidList []string) { defer wg.Done() - if deleteResults, deleteErr := operation.DeleteFilesAtOneVolumeServer(server, c.env.option.GrpcDialOption, fidList, false); deleteErr != nil { + if deleteResults, deleteErr := operation.DeleteFileIdsAtOneVolumeServer(server, c.env.option.GrpcDialOption, fidList, false); deleteErr != nil { err = deleteErr } else if deleteResults != nil { resultChan <- deleteResults diff --git a/weed/shell/command_volume_grow.go b/weed/shell/command_volume_grow.go new file mode 100644 index 000000000..21d98dddd --- /dev/null +++ b/weed/shell/command_volume_grow.go @@ -0,0 +1,64 @@ +package shell + +import ( + "context" + "flag" + "fmt" + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "io" +) + +func init() { + Commands = append(Commands, &commandGrow{}) +} + +type commandGrow struct { +} + +func (c *commandGrow) Name() string { + return "volume.grow" +} + +func (c *commandGrow) Help() string { + return `grow volumes + + volume.grow [-collection=<collection name>] [-dataCenter=<data center name>] + +` +} + +func (c *commandGrow) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + + volumeVacuumCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + growCount := volumeVacuumCommand.Uint("count", 2, "") + collection := volumeVacuumCommand.String("collection", "", "grow this collection") + dataCenter := volumeVacuumCommand.String("dataCenter", "", "grow volumes only from the specified data center") + + if err = volumeVacuumCommand.Parse(args); err != nil { + return nil + } + + assignRequest := &master_pb.AssignRequest{ + Count: 0, + Collection: *collection, + WritableVolumeCount: uint32(*growCount), + } + if *dataCenter != "" { + assignRequest.DataCenter = *dataCenter + } + + err = commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error { + _, err := client.Assign(context.Background(), assignRequest) + + if err != nil { + return fmt.Errorf("Assign: %v", err) + } + return nil + }) + + if err != nil { + return + } + + return nil +} diff --git a/weed/shell/command_volume_server_evacuate.go b/weed/shell/command_volume_server_evacuate.go index 57eb6fc45..bad695cd7 100644 --- a/weed/shell/command_volume_server_evacuate.go +++ b/weed/shell/command_volume_server_evacuate.go @@ -3,14 +3,15 @@ package shell import ( "flag" "fmt" + "io" + "os" + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/super_block" "github.com/seaweedfs/seaweedfs/weed/storage/types" "golang.org/x/exp/slices" - "io" - "os" ) func init() { @@ -219,7 +220,7 @@ func moveAwayOneNormalVolume(commandEnv *CommandEnv, volumeReplicas map[uint32][ }) for i := 0; i < len(otherNodes); i++ { emptyNode := otherNodes[i] - if freeVolumeCountfn(emptyNode.info) < 0 { + if freeVolumeCountfn(emptyNode.info) <= 0 { continue } hasMoved, err = maybeMoveOneVolume(commandEnv, volumeReplicas, thisNode, vol, emptyNode, applyChange) diff --git a/weed/shell/command_volume_tier_upload.go b/weed/shell/command_volume_tier_upload.go index 6932317ab..cb805b0cf 100644 --- a/weed/shell/command_volume_tier_upload.go +++ b/weed/shell/command_volume_tier_upload.go @@ -113,11 +113,14 @@ func doVolumeTierUpload(commandEnv *CommandEnv, writer io.Writer, collection str return fmt.Errorf("copy dat file for volume %d on %s to %s: %v", vid, existingLocations[0].Url, dest, err) } + if keepLocalDatFile { + return nil + } // now the first replica has the .idx and .vif files. // ask replicas on other volume server to delete its own local copy for i, location := range existingLocations { if i == 0 { - break + continue } fmt.Printf("delete volume %d from %s\n", vid, location.Url) err = deleteVolume(commandEnv.option.GrpcDialOption, vid, location.ServerAddress(), false) diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go index 4393c280c..134485946 100644 --- a/weed/stats/metrics.go +++ b/weed/stats/metrics.go @@ -76,7 +76,15 @@ var ( Subsystem: "master", Name: "volume_layout_total", Help: "Number of volumes in volume layouts", - }, []string{"collection", "replica", "type"}) + }, []string{"collection", "dataCenter", "type"}) + + MasterPickForWriteErrorCounter = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: "master", + Name: "pick_for_write_error", + Help: "Counter of master pick for write error", + }) MasterLeaderChangeCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go index b7fa82197..dd78735d2 100644 --- a/weed/storage/disk_location.go +++ b/weed/storage/disk_location.go @@ -60,6 +60,7 @@ func GenerateDirUuid(dir string) (dirUuidString string, err error) { } func NewDiskLocation(dir string, maxVolumeCount int32, minFreeSpace util.MinFreeSpace, idxDir string, diskType types.DiskType) *DiskLocation { + glog.V(4).Infof("Added new Disk %s: maxVolumes=%d", dir, maxVolumeCount) dir = util.ResolvePath(dir) if idxDir == "" { idxDir = dir @@ -138,7 +139,7 @@ func (l *DiskLocation) loadExistingVolume(dirEntry os.DirEntry, needleMapKind Ne // skip if ec volumes exists if skipIfEcVolumesExists { - if util.FileExists(l.Directory + "/" + volumeName + ".ecx") { + if util.FileExists(l.IdxDirectory + "/" + volumeName + ".ecx") { return false } } @@ -417,7 +418,6 @@ func (l *DiskLocation) LocateVolume(vid needle.VolumeId) (os.DirEntry, bool) { } func (l *DiskLocation) UnUsedSpace(volumeSizeLimit uint64) (unUsedSpace uint64) { - l.volumesLock.RLock() defer l.volumesLock.RUnlock() @@ -426,7 +426,11 @@ func (l *DiskLocation) UnUsedSpace(volumeSizeLimit uint64) (unUsedSpace uint64) continue } datSize, idxSize, _ := vol.FileStat() - unUsedSpace += volumeSizeLimit - (datSize + idxSize) + unUsedSpaceVolume := int64(volumeSizeLimit) - int64(datSize+idxSize) + glog.V(4).Infof("Volume stats for %d: volumeSizeLimit=%d, datSize=%d idxSize=%d unused=%d", vol.Id, volumeSizeLimit, datSize, idxSize, unUsedSpaceVolume) + if unUsedSpaceVolume >= 0 { + unUsedSpace += uint64(unUsedSpaceVolume) + } } return diff --git a/weed/storage/disk_location_test.go b/weed/storage/disk_location_test.go new file mode 100644 index 000000000..d105a477f --- /dev/null +++ b/weed/storage/disk_location_test.go @@ -0,0 +1,78 @@ +package storage + +import ( + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/storage/backend" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +type ( + mockBackendStorageFile struct { + backend.DiskFile + + datSize int64 + } +) + +func (df *mockBackendStorageFile) GetStat() (datSize int64, modTime time.Time, err error) { + return df.datSize, time.Now(), nil +} + +type ( + mockNeedleMapper struct { + NeedleMap + + idxSize uint64 + } +) + +func (nm *mockNeedleMapper) IndexFileSize() (idxSize uint64) { + return nm.idxSize +} + +func TestUnUsedSpace(t *testing.T) { + minFreeSpace := util.MinFreeSpace{Type: util.AsPercent, Percent: 1, Raw: "1"} + + diskLocation := DiskLocation{ + Directory: "/test/", + DirectoryUuid: "1234", + IdxDirectory: "/test/", + DiskType: "hdd", + MaxVolumeCount: 0, + OriginalMaxVolumeCount: 0, + MinFreeSpace: minFreeSpace, + } + diskLocation.volumes = make(map[needle.VolumeId]*Volume) + + volumes := [3]*Volume{ + {dir: diskLocation.Directory, dirIdx: diskLocation.IdxDirectory, Collection: "", Id: 0, DataBackend: &mockBackendStorageFile{datSize: 990}, nm: &mockNeedleMapper{idxSize: 10}}, + {dir: diskLocation.Directory, dirIdx: diskLocation.IdxDirectory, Collection: "", Id: 1, DataBackend: &mockBackendStorageFile{datSize: 990}, nm: &mockNeedleMapper{idxSize: 10}}, + {dir: diskLocation.Directory, dirIdx: diskLocation.IdxDirectory, Collection: "", Id: 2, DataBackend: &mockBackendStorageFile{datSize: 990}, nm: &mockNeedleMapper{idxSize: 10}}, + } + + for i, vol := range volumes { + diskLocation.SetVolume(needle.VolumeId(i), vol) + } + + // Testing when there's still space + unUsedSpace := diskLocation.UnUsedSpace(1200) + if unUsedSpace != 600 { + t.Errorf("unUsedSpace incorrect: %d != %d", unUsedSpace, 1500) + } + + // Testing when there's exactly 0 space + unUsedSpace = diskLocation.UnUsedSpace(1000) + if unUsedSpace != 0 { + t.Errorf("unUsedSpace incorrect: %d != %d", unUsedSpace, 0) + } + + // Testing when there's negative free space + unUsedSpace = diskLocation.UnUsedSpace(900) + if unUsedSpace != 0 { + t.Errorf("unUsedSpace incorrect: %d != %d", unUsedSpace, 0) + } + +} diff --git a/weed/storage/needle/needle_parse_upload.go b/weed/storage/needle/needle_parse_upload.go index b512c16a2..28de27910 100644 --- a/weed/storage/needle/needle_parse_upload.go +++ b/weed/storage/needle/needle_parse_upload.go @@ -43,7 +43,7 @@ func ParseUpload(r *http.Request, sizeLimit int64, bytesBuffer *bytes.Buffer) (p } } - if r.Method == "POST" { + if r.Method == http.MethodPost { contentType := r.Header.Get("Content-Type") // If content-type is explicitly set, upload the file without parsing form-data diff --git a/weed/topology/data_node.go b/weed/topology/data_node.go index 5e48056f7..715b5ed80 100644 --- a/weed/topology/data_node.go +++ b/weed/topology/data_node.go @@ -135,12 +135,12 @@ func (dn *DataNode) DeltaUpdateVolumes(newVolumes, deletedVolumes []storage.Volu } func (dn *DataNode) AdjustMaxVolumeCounts(maxVolumeCounts map[string]uint32) { - deltaDiskUsages := newDiskUsages() for diskType, maxVolumeCount := range maxVolumeCounts { if maxVolumeCount == 0 { // the volume server may have set the max to zero continue } + deltaDiskUsages := newDiskUsages() dt := types.ToDiskType(diskType) currentDiskUsage := dn.diskUsages.getOrCreateDisk(dt) currentDiskUsageMaxVolumeCount := atomic.LoadInt64(¤tDiskUsage.maxVolumeCount) diff --git a/weed/topology/store_replicate.go b/weed/topology/store_replicate.go index 82c2db79c..b4a7d649c 100644 --- a/weed/topology/store_replicate.go +++ b/weed/topology/store_replicate.go @@ -20,6 +20,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/storage/types" "github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util/buffer_pool" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOption, s *storage.Store, volumeId needle.VolumeId, n *needle.Needle, r *http.Request, contentMd5 string) (isUnchanged bool, err error) { @@ -105,7 +106,12 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt BytesBuffer: bytesBuffer, } - _, err := operation.UploadData(n.Data, uploadOption) + uploader, err := operation.NewUploader() + if err != nil { + glog.Errorf("replication-UploadData, err:%v, url:%s", err, u.String()) + return err + } + _, err = uploader.UploadData(n.Data, uploadOption) if err != nil { glog.Errorf("replication-UploadData, err:%v, url:%s", err, u.String()) } @@ -144,7 +150,7 @@ func ReplicatedDelete(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOp if len(remoteLocations) > 0 { //send to other replica locations if err = DistributedOperation(remoteLocations, func(location operation.Location) error { - return util.Delete("http://"+location.Url+r.URL.Path+"?type=replicate", string(jwt)) + return util_http.Delete("http://"+location.Url+r.URL.Path+"?type=replicate", string(jwt)) }); err != nil { size = 0 } diff --git a/weed/topology/topology.go b/weed/topology/topology.go index 0a4cb4050..6a149bd56 100644 --- a/weed/topology/topology.go +++ b/weed/topology/topology.go @@ -50,8 +50,11 @@ type Topology struct { RaftServer raft.Server RaftServerAccessLock sync.RWMutex HashicorpRaft *hashicorpRaft.Raft - UuidAccessLock sync.RWMutex - UuidMap map[string][]string + barrierLock sync.Mutex + barrierDone bool + + UuidAccessLock sync.RWMutex + UuidMap map[string][]string } func NewTopology(id string, seq sequence.Sequencer, volumeSizeLimit uint64, pulse int, replicationAsMin bool) *Topology { @@ -120,6 +123,42 @@ func (t *Topology) IsLeader() bool { return false } +func (t *Topology) IsLeaderAndCanRead() bool { + if t.RaftServer != nil { + return t.IsLeader() + } else if t.HashicorpRaft != nil { + return t.IsLeader() && t.DoBarrier() + } else { + return false + } +} + +func (t *Topology) DoBarrier() bool { + t.barrierLock.Lock() + defer t.barrierLock.Unlock() + if t.barrierDone { + return true + } + + glog.V(0).Infof("raft do barrier") + barrier := t.HashicorpRaft.Barrier(2 * time.Minute) + if err := barrier.Error(); err != nil { + glog.Errorf("failed to wait for barrier, error %s", err) + return false + + } + + t.barrierDone = true + glog.V(0).Infof("raft do barrier success") + return true +} + +func (t *Topology) BarrierReset() { + t.barrierLock.Lock() + defer t.barrierLock.Unlock() + t.barrierDone = false +} + func (t *Topology) Leader() (l pb.ServerAddress, err error) { exponentialBackoff := backoff.NewExponentialBackOff() exponentialBackoff.InitialInterval = 100 * time.Millisecond @@ -180,6 +219,10 @@ func (t *Topology) Lookup(collection string, vid needle.VolumeId) (dataNodes []* } func (t *Topology) NextVolumeId() (needle.VolumeId, error) { + if !t.IsLeaderAndCanRead() { + return 0, fmt.Errorf("as leader can not read yet") + + } vid := t.GetMaxVolumeId() next := vid.Next() @@ -208,8 +251,8 @@ func (t *Topology) PickForWrite(requestedCount uint64, option *VolumeGrowOption, if err != nil { return "", 0, nil, shouldGrow, fmt.Errorf("failed to find writable volumes for collection:%s replication:%s ttl:%s error: %v", option.Collection, option.ReplicaPlacement.String(), option.Ttl.String(), err) } - if volumeLocationList.Length() == 0 { - return "", 0, nil, shouldGrow, fmt.Errorf("no writable volumes available for collection:%s replication:%s ttl:%s", option.Collection, option.ReplicaPlacement.String(), option.Ttl.String()) + if volumeLocationList == nil || volumeLocationList.Length() == 0 { + return "", 0, nil, shouldGrow, fmt.Errorf("%s available for collection:%s replication:%s ttl:%s", noWritableVolumes, option.Collection, option.ReplicaPlacement.String(), option.Ttl.String()) } nextFileId := t.Sequence.NextFileId(requestedCount) fileId = needle.NewFileId(vid, nextFileId, rand.Uint32()).String() @@ -285,6 +328,22 @@ func (t *Topology) UnRegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) { } } +func (t *Topology) DataCenterExists(dcName string) bool { + return dcName == "" || t.GetOrCreateDataCenter(dcName) != nil +} + +func (t *Topology) GetDataCenter(dcName string) (dc *DataCenter) { + t.RLock() + defer t.RUnlock() + for _, c := range t.children { + dc = c.(*DataCenter) + if string(dc.Id()) == dcName { + return dc + } + } + return dc +} + func (t *Topology) GetOrCreateDataCenter(dcName string) *DataCenter { t.Lock() defer t.Unlock() diff --git a/weed/topology/topology_event_handling.go b/weed/topology/topology_event_handling.go index 761649ff4..d0ecd089a 100644 --- a/weed/topology/topology_event_handling.go +++ b/weed/topology/topology_event_handling.go @@ -90,6 +90,11 @@ func (t *Topology) UnRegisterDataNode(dn *DataNode) { vl.SetVolumeUnavailable(dn, v.Id) } + // unregister ec shards when volume server disconnected + for _, s := range dn.GetEcShards() { + t.UnRegisterEcShards(s, dn) + } + negativeUsages := dn.GetDiskUsages().negative() dn.UpAdjustDiskUsageDelta(negativeUsages) dn.DeltaUpdateVolumes([]storage.VolumeInfo{}, dn.GetVolumes()) diff --git a/weed/topology/topology_info.go b/weed/topology/topology_info.go index 120ae0d42..89f9097f6 100644 --- a/weed/topology/topology_info.go +++ b/weed/topology/topology_info.go @@ -42,6 +42,15 @@ func (t *Topology) ToInfo() (info TopologyInfo) { return } +func (t *Topology) ListVolumeLyauts() (volumeLayouts []*VolumeLayout) { + for _, col := range t.collectionMap.Items() { + for _, volumeLayout := range col.(*Collection).storageType2VolumeLayout.Items() { + volumeLayouts = append(volumeLayouts, volumeLayout.(*VolumeLayout)) + } + } + return volumeLayouts +} + func (t *Topology) ToVolumeMap() interface{} { m := make(map[string]interface{}) m["Max"] = t.diskUsages.GetMaxVolumeCount() diff --git a/weed/topology/volume_growth.go b/weed/topology/volume_growth.go index cfc31c8b1..ff516599d 100644 --- a/weed/topology/volume_growth.go +++ b/weed/topology/volume_growth.go @@ -27,14 +27,14 @@ This package is created to resolve these replica placement issues: type VolumeGrowRequest struct { Option *VolumeGrowOption - Count int + Count uint32 } type volumeGrowthStrategy struct { - Copy1Count int - Copy2Count int - Copy3Count int - CopyOtherCount int + Copy1Count uint32 + Copy2Count uint32 + Copy3Count uint32 + CopyOtherCount uint32 Threshold float64 } @@ -75,7 +75,7 @@ func NewDefaultVolumeGrowth() *VolumeGrowth { // one replication type may need rp.GetCopyCount() actual volumes // given copyCount, how many logical volumes to create -func (vg *VolumeGrowth) findVolumeCount(copyCount int) (count int) { +func (vg *VolumeGrowth) findVolumeCount(copyCount int) (count uint32) { switch copyCount { case 1: count = VolumeGrowStrategy.Copy1Count @@ -89,7 +89,7 @@ func (vg *VolumeGrowth) findVolumeCount(copyCount int) (count int) { return } -func (vg *VolumeGrowth) AutomaticGrowByType(option *VolumeGrowOption, grpcDialOption grpc.DialOption, topo *Topology, targetCount int) (result []*master_pb.VolumeLocation, err error) { +func (vg *VolumeGrowth) AutomaticGrowByType(option *VolumeGrowOption, grpcDialOption grpc.DialOption, topo *Topology, targetCount uint32) (result []*master_pb.VolumeLocation, err error) { if targetCount == 0 { targetCount = vg.findVolumeCount(option.ReplicaPlacement.GetCopyCount()) } @@ -99,11 +99,11 @@ func (vg *VolumeGrowth) AutomaticGrowByType(option *VolumeGrowOption, grpcDialOp } return result, err } -func (vg *VolumeGrowth) GrowByCountAndType(grpcDialOption grpc.DialOption, targetCount int, option *VolumeGrowOption, topo *Topology) (result []*master_pb.VolumeLocation, err error) { +func (vg *VolumeGrowth) GrowByCountAndType(grpcDialOption grpc.DialOption, targetCount uint32, option *VolumeGrowOption, topo *Topology) (result []*master_pb.VolumeLocation, err error) { vg.accessLock.Lock() defer vg.accessLock.Unlock() - for i := 0; i < targetCount; i++ { + for i := uint32(0); i < targetCount; i++ { if res, e := vg.findAndGrow(grpcDialOption, topo, option); e == nil { result = append(result, res...) } else { diff --git a/weed/topology/volume_layout.go b/weed/topology/volume_layout.go index 66f7118c9..e2a360165 100644 --- a/weed/topology/volume_layout.go +++ b/weed/topology/volume_layout.go @@ -1,7 +1,6 @@ package topology import ( - "errors" "fmt" "github.com/seaweedfs/seaweedfs/weed/stats" "math/rand" @@ -28,9 +27,10 @@ const ( type volumeState string const ( - readOnlyState volumeState = "ReadOnly" - oversizedState = "Oversized" - crowdedState = "Crowded" + readOnlyState volumeState = "ReadOnly" + oversizedState = "Oversized" + crowdedState = "Crowded" + noWritableVolumes = "No writable volumes" ) type stateIndicator func(copyState) bool @@ -106,7 +106,8 @@ func (v *volumesBinaryState) copyState(list *VolumeLocationList) copyState { // mapping from volume to its locations, inverted from server to volume type VolumeLayout struct { - growRequestCount int32 + growRequest atomic.Bool + lastGrowCount atomic.Uint32 rp *super_block.ReplicaPlacement ttl *needle.TTL diskType types.DiskType @@ -292,23 +293,15 @@ func (vl *VolumeLayout) PickForWrite(count uint64, option *VolumeGrowOption) (vi lenWriters := len(vl.writables) if lenWriters <= 0 { - //glog.V(0).Infoln("No more writable volumes!") - shouldGrow = true - return 0, 0, nil, shouldGrow, errors.New("No more writable volumes!") + return 0, 0, nil, true, fmt.Errorf("%s in volume layout", noWritableVolumes) } if option.DataCenter == "" && option.Rack == "" && option.DataNode == "" { vid := vl.writables[rand.Intn(lenWriters)] locationList = vl.vid2location[vid] - if locationList != nil && locationList.Length() > 0 { - // check whether picked file is close to full - dn := locationList.Head() - info, _ := dn.GetVolumesById(vid) - if float64(info.Size) > float64(vl.volumeSizeLimit)*VolumeGrowStrategy.Threshold { - shouldGrow = true - } - return vid, count, locationList.Copy(), shouldGrow, nil + if locationList == nil || len(locationList.list) == 0 { + return 0, 0, nil, false, fmt.Errorf("Strangely vid %s is on no machine!", vid.String()) } - return 0, 0, nil, shouldGrow, errors.New("Strangely vid " + vid.String() + " is on no machine!") + return vid, count, locationList.Copy(), false, nil } // clone vl.writables @@ -331,34 +324,38 @@ func (vl *VolumeLayout) PickForWrite(count uint64, option *VolumeGrowOption) (vi if option.DataNode != "" && dn.Id() != NodeId(option.DataNode) { continue } - vid, locationList = writableVolumeId, volumeLocationList.Copy() - // check whether picked file is close to full - info, _ := dn.GetVolumesById(writableVolumeId) - if float64(info.Size) > float64(vl.volumeSizeLimit)*VolumeGrowStrategy.Threshold { - shouldGrow = true - } - counter = count + vid, locationList, counter = writableVolumeId, volumeLocationList.Copy(), count return } } - return vid, count, locationList, true, fmt.Errorf("No writable volumes in DataCenter:%v Rack:%v DataNode:%v", option.DataCenter, option.Rack, option.DataNode) + return vid, count, locationList, true, fmt.Errorf("%s in DataCenter:%v Rack:%v DataNode:%v", noWritableVolumes, option.DataCenter, option.Rack, option.DataNode) } func (vl *VolumeLayout) HasGrowRequest() bool { - return atomic.LoadInt32(&vl.growRequestCount) > 0 + return vl.growRequest.Load() } func (vl *VolumeLayout) AddGrowRequest() { - atomic.AddInt32(&vl.growRequestCount, 1) + vl.growRequest.Store(true) } func (vl *VolumeLayout) DoneGrowRequest() { - atomic.AddInt32(&vl.growRequestCount, -1) + vl.growRequest.Store(false) +} + +func (vl *VolumeLayout) SetLastGrowCount(count uint32) { + if vl.lastGrowCount.Load() != count { + vl.lastGrowCount.Store(count) + } +} + +func (vl *VolumeLayout) GetLastGrowCount() uint32 { + return vl.lastGrowCount.Load() } func (vl *VolumeLayout) ShouldGrowVolumes(option *VolumeGrowOption) bool { total, active, crowded := vl.GetActiveVolumeCount(option) - stats.MasterVolumeLayout.WithLabelValues(option.Collection, option.ReplicaPlacement.String(), "total").Set(float64(total)) - stats.MasterVolumeLayout.WithLabelValues(option.Collection, option.ReplicaPlacement.String(), "active").Set(float64(active)) - stats.MasterVolumeLayout.WithLabelValues(option.Collection, option.ReplicaPlacement.String(), "crowded").Set(float64(crowded)) + stats.MasterVolumeLayout.WithLabelValues(option.Collection, option.DataCenter, "total").Set(float64(total)) + stats.MasterVolumeLayout.WithLabelValues(option.Collection, option.DataCenter, "active").Set(float64(active)) + stats.MasterVolumeLayout.WithLabelValues(option.Collection, option.DataCenter, "crowded").Set(float64(crowded)) //glog.V(0).Infof("active volume: %d, high usage volume: %d\n", active, high) return active <= crowded } @@ -539,6 +536,14 @@ func (vl *VolumeLayout) ToInfo() (info VolumeLayoutInfo) { return } +func (vl *VolumeLayout) ToGrowOption() (option *VolumeGrowOption) { + option = &VolumeGrowOption{} + option.ReplicaPlacement = vl.rp + option.Ttl = vl.ttl + option.DiskType = vl.diskType + return +} + func (vl *VolumeLayout) Stats() *VolumeLayoutStats { vl.accessLock.RLock() defer vl.accessLock.RUnlock() diff --git a/weed/util/bytes.go b/weed/util/bytes.go index 26da91033..482dc3a48 100644 --- a/weed/util/bytes.go +++ b/weed/util/bytes.go @@ -148,6 +148,12 @@ func RandomInt32() int32 { return int32(BytesToUint32(buf)) } +func RandomUint64() int32 { + buf := make([]byte, 8) + rand.Read(buf) + return int32(BytesToUint64(buf)) +} + func RandomBytes(byteCount int) []byte { buf := make([]byte, byteCount) rand.Read(buf) diff --git a/weed/util/config.go b/weed/util/config.go index a242742fc..7ac765ff5 100644 --- a/weed/util/config.go +++ b/weed/util/config.go @@ -11,6 +11,7 @@ import ( var ( ConfigurationFileDirectory DirectoryValueType + loadSecurityConfigOnce sync.Once ) type DirectoryValueType string @@ -31,6 +32,12 @@ type Configuration interface { SetDefault(key string, value interface{}) } +func LoadSecurityConfiguration(){ + loadSecurityConfigOnce.Do(func() { + LoadConfiguration("security", false) + }) +} + func LoadConfiguration(configFileName string, required bool) (loaded bool) { // find a filer store diff --git a/weed/util/constants.go b/weed/util/constants.go index 09f813e5b..7911d03f6 100644 --- a/weed/util/constants.go +++ b/weed/util/constants.go @@ -7,7 +7,9 @@ import ( const HttpStatusCancelled = 499 var ( - VERSION_NUMBER = fmt.Sprintf("%.02f", 3.68) + MAJOR_VERSION = int32(3) + MINOR_VERSION = int32(71) + VERSION_NUMBER = fmt.Sprintf("%d.%02d", MAJOR_VERSION, MINOR_VERSION) VERSION = sizeLimit + " " + VERSION_NUMBER COMMIT = "" ) diff --git a/weed/util/http/client/http_client.go b/weed/util/http/client/http_client.go new file mode 100644 index 000000000..d1d2f5c56 --- /dev/null +++ b/weed/util/http/client/http_client.go @@ -0,0 +1,201 @@ +package client + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + util "github.com/seaweedfs/seaweedfs/weed/util" + "github.com/spf13/viper" + "io" + "net/http" + "net/url" + "os" + "strings" + "sync" +) + +var ( + loadSecurityConfigOnce sync.Once +) + +type HTTPClient struct { + Client *http.Client + Transport *http.Transport + expectHttpsScheme bool +} + +func (httpClient *HTTPClient) Do(req *http.Request) (*http.Response, error) { + req.URL.Scheme = httpClient.GetHttpScheme() + return httpClient.Client.Do(req) +} + +func (httpClient *HTTPClient) Get(url string) (resp *http.Response, err error) { + url, err = httpClient.NormalizeHttpScheme(url) + if err != nil { + return nil, err + } + return httpClient.Client.Get(url) +} + +func (httpClient *HTTPClient) Post(url, contentType string, body io.Reader) (resp *http.Response, err error) { + url, err = httpClient.NormalizeHttpScheme(url) + if err != nil { + return nil, err + } + return httpClient.Client.Post(url, contentType, body) +} + +func (httpClient *HTTPClient) PostForm(url string, data url.Values) (resp *http.Response, err error) { + url, err = httpClient.NormalizeHttpScheme(url) + if err != nil { + return nil, err + } + return httpClient.Client.PostForm(url, data) +} + +func (httpClient *HTTPClient) Head(url string) (resp *http.Response, err error) { + url, err = httpClient.NormalizeHttpScheme(url) + if err != nil { + return nil, err + } + return httpClient.Client.Head(url) +} +func (httpClient *HTTPClient) CloseIdleConnections() { + httpClient.Client.CloseIdleConnections() +} + +func (httpClient *HTTPClient) GetClientTransport() *http.Transport { + return httpClient.Transport +} + +func (httpClient *HTTPClient) GetHttpScheme() string { + if httpClient.expectHttpsScheme { + return "https" + } + return "http" +} + +func (httpClient *HTTPClient) NormalizeHttpScheme(rawURL string) (string, error) { + expectedScheme := httpClient.GetHttpScheme() + + if !(strings.HasPrefix(rawURL, "http://") || strings.HasPrefix(rawURL, "https://")) { + return expectedScheme + "://" + rawURL, nil + } + + parsedURL, err := url.Parse(rawURL) + if err != nil { + return "", err + } + + if expectedScheme != parsedURL.Scheme { + parsedURL.Scheme = expectedScheme + } + return parsedURL.String(), nil +} + +func NewHttpClient(clientName ClientName, opts ...HttpClientOpt) (*HTTPClient, error) { + httpClient := HTTPClient{} + httpClient.expectHttpsScheme = checkIsHttpsClientEnabled(clientName) + var tlsConfig *tls.Config = nil + + if httpClient.expectHttpsScheme { + clientCertPair, err := getClientCertPair(clientName) + if err != nil { + return nil, err + } + + clientCaCert, clientCaCertName, err := getClientCaCert(clientName) + if err != nil { + return nil, err + } + + if clientCertPair != nil || len(clientCaCert) != 0 { + caCertPool, err := createHTTPClientCertPool(clientCaCert, clientCaCertName) + if err != nil { + return nil, err + } + + tlsConfig = &tls.Config{ + Certificates: []tls.Certificate{}, + RootCAs: caCertPool, + InsecureSkipVerify: false, + } + + if clientCertPair != nil { + tlsConfig.Certificates = append(tlsConfig.Certificates, *clientCertPair) + } + } + } + + httpClient.Transport = &http.Transport{ + MaxIdleConns: 1024, + MaxIdleConnsPerHost: 1024, + TLSClientConfig: tlsConfig, + } + httpClient.Client = &http.Client{ + Transport: httpClient.Transport, + } + + for _, opt := range opts { + opt(&httpClient) + } + return &httpClient, nil +} + +func getStringOptionFromSecurityConfiguration(clientName ClientName, stringOptionName string) string { + util.LoadSecurityConfiguration() + return viper.GetString(fmt.Sprintf("https.%s.%s", clientName.LowerCaseString(), stringOptionName)) +} + +func getBoolOptionFromSecurityConfiguration(clientName ClientName, boolOptionName string) bool { + util.LoadSecurityConfiguration() + return viper.GetBool(fmt.Sprintf("https.%s.%s", clientName.LowerCaseString(), boolOptionName)) +} + +func checkIsHttpsClientEnabled(clientName ClientName) bool { + return getBoolOptionFromSecurityConfiguration(clientName, "enabled") +} + +func getFileContentFromSecurityConfiguration(clientName ClientName, fileType string) ([]byte, string, error) { + if fileName := getStringOptionFromSecurityConfiguration(clientName, fileType); fileName != "" { + fileContent, err := os.ReadFile(fileName) + if err != nil { + return nil, fileName, err + } + return fileContent, fileName, err + } + return nil, "", nil +} + +func getClientCertPair(clientName ClientName) (*tls.Certificate, error) { + certFileName := getStringOptionFromSecurityConfiguration(clientName, "cert") + keyFileName := getStringOptionFromSecurityConfiguration(clientName, "key") + if certFileName == "" && keyFileName == "" { + return nil, nil + } + if certFileName != "" && keyFileName != "" { + clientCert, err := tls.LoadX509KeyPair(certFileName, keyFileName) + if err != nil { + return nil, fmt.Errorf("error loading client certificate and key: %s", err) + } + return &clientCert, nil + } + return nil, fmt.Errorf("error loading key pair: key `%s` and certificate `%s`", keyFileName, certFileName) +} + +func getClientCaCert(clientName ClientName) ([]byte, string, error) { + return getFileContentFromSecurityConfiguration(clientName, "ca") +} + +func createHTTPClientCertPool(certContent []byte, fileName string) (*x509.CertPool, error) { + certPool := x509.NewCertPool() + if len(certContent) == 0 { + return certPool, nil + } + + ok := certPool.AppendCertsFromPEM(certContent) + if !ok { + return nil, fmt.Errorf("error processing certificate in %s", fileName) + } + return certPool, nil +} diff --git a/weed/util/http/client/http_client_interface.go b/weed/util/http/client/http_client_interface.go new file mode 100644 index 000000000..7a2d43360 --- /dev/null +++ b/weed/util/http/client/http_client_interface.go @@ -0,0 +1,16 @@ +package client + +import ( + "io" + "net/http" + "net/url" +) + +type HTTPClientInterface interface { + Do(req *http.Request) (*http.Response, error) + Get(url string) (resp *http.Response, err error) + Post(url, contentType string, body io.Reader) (resp *http.Response, err error) + PostForm(url string, data url.Values) (resp *http.Response, err error) + Head(url string) (resp *http.Response, err error) + CloseIdleConnections() +} diff --git a/weed/util/http/client/http_client_name.go b/weed/util/http/client/http_client_name.go new file mode 100644 index 000000000..aedaebbc6 --- /dev/null +++ b/weed/util/http/client/http_client_name.go @@ -0,0 +1,14 @@ +package client + +import "strings" + +type ClientName int + +//go:generate stringer -type=ClientName -output=http_client_name_string.go +const ( + Client ClientName = iota +) + +func (name *ClientName) LowerCaseString() string { + return strings.ToLower(name.String()) +} diff --git a/weed/util/http/client/http_client_name_string.go b/weed/util/http/client/http_client_name_string.go new file mode 100644 index 000000000..652fcdaac --- /dev/null +++ b/weed/util/http/client/http_client_name_string.go @@ -0,0 +1,23 @@ +// Code generated by "stringer -type=ClientName -output=http_client_name_string.go"; DO NOT EDIT. + +package client + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[Client-0] +} + +const _ClientName_name = "Client" + +var _ClientName_index = [...]uint8{0, 6} + +func (i ClientName) String() string { + if i < 0 || i >= ClientName(len(_ClientName_index)-1) { + return "ClientName(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _ClientName_name[_ClientName_index[i]:_ClientName_index[i+1]] +} diff --git a/weed/util/http/client/http_client_opt.go b/weed/util/http/client/http_client_opt.go new file mode 100644 index 000000000..1ff9d533d --- /dev/null +++ b/weed/util/http/client/http_client_opt.go @@ -0,0 +1,18 @@ +package client + +import ( + "net" + "time" +) + +type HttpClientOpt = func(clientCfg *HTTPClient) + +func AddDialContext(httpClient *HTTPClient) { + dialContext := (&net.Dialer{ + Timeout: 10 * time.Second, + KeepAlive: 10 * time.Second, + }).DialContext + + httpClient.Transport.DialContext = dialContext + httpClient.Client.Transport = httpClient.Transport +} diff --git a/weed/util/http/http_global_client_init.go b/weed/util/http/http_global_client_init.go new file mode 100644 index 000000000..0dcb05cfd --- /dev/null +++ b/weed/util/http/http_global_client_init.go @@ -0,0 +1,27 @@ +package http + +import ( + "github.com/seaweedfs/seaweedfs/weed/glog" + util_http_client "github.com/seaweedfs/seaweedfs/weed/util/http/client" +) + +var ( + globalHttpClient *util_http_client.HTTPClient +) + +func NewGlobalHttpClient(opt ...util_http_client.HttpClientOpt) (*util_http_client.HTTPClient, error) { + return util_http_client.NewHttpClient(util_http_client.Client, opt...) +} + +func GetGlobalHttpClient() *util_http_client.HTTPClient { + return globalHttpClient +} + +func InitGlobalHttpClient() { + var err error + + globalHttpClient, err = NewGlobalHttpClient() + if err != nil { + glog.Fatalf("error init global http client: %v", err) + } +} diff --git a/weed/util/http_util.go b/weed/util/http/http_global_client_util.go index 6f6a17008..c3931a790 100644 --- a/weed/util/http_util.go +++ b/weed/util/http/http_global_client_util.go @@ -1,4 +1,4 @@ -package util +package http import ( "compress/gzip" @@ -6,6 +6,7 @@ import ( "errors" "fmt" "github.com/seaweedfs/seaweedfs/weed/util/mem" + "github.com/seaweedfs/seaweedfs/weed/util" "io" "net/http" "net/url" @@ -15,23 +16,8 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" ) -var ( - client *http.Client - Transport *http.Transport -) - -func init() { - Transport = &http.Transport{ - MaxIdleConns: 1024, - MaxIdleConnsPerHost: 1024, - } - client = &http.Client{ - Transport: Transport, - } -} - func Post(url string, values url.Values) ([]byte, error) { - r, err := client.PostForm(url, values) + r, err := GetGlobalHttpClient().PostForm(url, values) if err != nil { return nil, err } @@ -57,14 +43,14 @@ func Get(url string) ([]byte, bool, error) { } func GetAuthenticated(url, jwt string) ([]byte, bool, error) { - request, err := http.NewRequest("GET", url, nil) + request, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return nil, true, err } maybeAddAuth(request, jwt) request.Header.Add("Accept-Encoding", "gzip") - response, err := client.Do(request) + response, err := GetGlobalHttpClient().Do(request) if err != nil { return nil, true, err } @@ -94,7 +80,7 @@ func GetAuthenticated(url, jwt string) ([]byte, bool, error) { } func Head(url string) (http.Header, error) { - r, err := client.Head(url) + r, err := GetGlobalHttpClient().Head(url) if err != nil { return nil, err } @@ -112,12 +98,12 @@ func maybeAddAuth(req *http.Request, jwt string) { } func Delete(url string, jwt string) error { - req, err := http.NewRequest("DELETE", url, nil) + req, err := http.NewRequest(http.MethodDelete, url, nil) maybeAddAuth(req, jwt) if err != nil { return err } - resp, e := client.Do(req) + resp, e := GetGlobalHttpClient().Do(req) if e != nil { return e } @@ -140,12 +126,12 @@ func Delete(url string, jwt string) error { } func DeleteProxied(url string, jwt string) (body []byte, httpStatus int, err error) { - req, err := http.NewRequest("DELETE", url, nil) + req, err := http.NewRequest(http.MethodDelete, url, nil) maybeAddAuth(req, jwt) if err != nil { return } - resp, err := client.Do(req) + resp, err := GetGlobalHttpClient().Do(req) if err != nil { return } @@ -159,7 +145,7 @@ func DeleteProxied(url string, jwt string) (body []byte, httpStatus int, err err } func GetBufferStream(url string, values url.Values, allocatedBytes []byte, eachBuffer func([]byte)) error { - r, err := client.PostForm(url, values) + r, err := GetGlobalHttpClient().PostForm(url, values) if err != nil { return err } @@ -182,7 +168,7 @@ func GetBufferStream(url string, values url.Values, allocatedBytes []byte, eachB } func GetUrlStream(url string, values url.Values, readFn func(io.Reader) error) error { - r, err := client.PostForm(url, values) + r, err := GetGlobalHttpClient().PostForm(url, values) if err != nil { return err } @@ -194,14 +180,14 @@ func GetUrlStream(url string, values url.Values, readFn func(io.Reader) error) e } func DownloadFile(fileUrl string, jwt string) (filename string, header http.Header, resp *http.Response, e error) { - req, err := http.NewRequest("GET", fileUrl, nil) + req, err := http.NewRequest(http.MethodGet, fileUrl, nil) if err != nil { return "", nil, nil, err } maybeAddAuth(req, jwt) - response, err := client.Do(req) + response, err := GetGlobalHttpClient().Do(req) if err != nil { return "", nil, nil, err } @@ -219,14 +205,11 @@ func DownloadFile(fileUrl string, jwt string) (filename string, header http.Head } func Do(req *http.Request) (resp *http.Response, err error) { - return client.Do(req) + return GetGlobalHttpClient().Do(req) } -func NormalizeUrl(url string) string { - if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") { - return url - } - return "http://" + url +func NormalizeUrl(url string) (string, error) { + return GetGlobalHttpClient().NormalizeHttpScheme(url) } func ReadUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullChunk bool, offset int64, size int, buf []byte) (int64, error) { @@ -239,7 +222,7 @@ func ReadUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullC return int64(n), err } - req, err := http.NewRequest("GET", fileUrl, nil) + req, err := http.NewRequest(http.MethodGet, fileUrl, nil) if err != nil { return 0, err } @@ -249,7 +232,7 @@ func ReadUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullC req.Header.Set("Accept-Encoding", "gzip") } - r, err := client.Do(req) + r, err := GetGlobalHttpClient().Do(req) if err != nil { return 0, err } @@ -310,7 +293,7 @@ func ReadUrlAsStreamAuthenticated(fileUrl, jwt string, cipherKey []byte, isConte return readEncryptedUrl(fileUrl, jwt, cipherKey, isContentGzipped, isFullChunk, offset, size, fn) } - req, err := http.NewRequest("GET", fileUrl, nil) + req, err := http.NewRequest(http.MethodGet, fileUrl, nil) maybeAddAuth(req, jwt) if err != nil { return false, err @@ -322,7 +305,7 @@ func ReadUrlAsStreamAuthenticated(fileUrl, jwt string, cipherKey []byte, isConte req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size)-1)) } - r, err := client.Do(req) + r, err := GetGlobalHttpClient().Do(req) if err != nil { return true, err } @@ -368,12 +351,12 @@ func readEncryptedUrl(fileUrl, jwt string, cipherKey []byte, isContentCompressed if err != nil { return retryable, fmt.Errorf("fetch %s: %v", fileUrl, err) } - decryptedData, err := Decrypt(encryptedData, CipherKey(cipherKey)) + decryptedData, err := util.Decrypt(encryptedData, util.CipherKey(cipherKey)) if err != nil { return false, fmt.Errorf("decrypt %s: %v", fileUrl, err) } if isContentCompressed { - decryptedData, err = DecompressData(decryptedData) + decryptedData, err = util.DecompressData(decryptedData) if err != nil { glog.V(0).Infof("unzip decrypt %s: %v", fileUrl, err) } @@ -391,7 +374,7 @@ func readEncryptedUrl(fileUrl, jwt string, cipherKey []byte, isContentCompressed func ReadUrlAsReaderCloser(fileUrl string, jwt string, rangeHeader string) (*http.Response, io.ReadCloser, error) { - req, err := http.NewRequest("GET", fileUrl, nil) + req, err := http.NewRequest(http.MethodGet, fileUrl, nil) if err != nil { return nil, nil, err } @@ -403,7 +386,7 @@ func ReadUrlAsReaderCloser(fileUrl string, jwt string, rangeHeader string) (*htt maybeAddAuth(req, jwt) - r, err := client.Do(req) + r, err := GetGlobalHttpClient().Do(req) if err != nil { return nil, nil, err } @@ -463,7 +446,7 @@ func RetriedFetchChunkData(buffer []byte, urlStrings []string, cipherKey []byte, var shouldRetry bool - for waitTime := time.Second; waitTime < RetryWaitTime; waitTime += waitTime / 2 { + for waitTime := time.Second; waitTime < util.RetryWaitTime; waitTime += waitTime / 2 { for _, urlString := range urlStrings { n = 0 if strings.Contains(urlString, "%") { @@ -494,4 +477,4 @@ func RetriedFetchChunkData(buffer []byte, urlStrings []string, cipherKey []byte, return n, err -} +}
\ No newline at end of file diff --git a/weed/util/queue.go b/weed/util/queue.go index 1e6211e0d..69efc078f 100644 --- a/weed/util/queue.go +++ b/weed/util/queue.go @@ -1,35 +1,37 @@ package util -import "sync" +import ( + "sync" +) -type node struct { - data interface{} - next *node +type node[T any]struct { + data T + next *node[T] } -type Queue struct { - head *node - tail *node +type Queue[T any] struct { + head *node[T] + tail *node[T] count int sync.RWMutex } -func NewQueue() *Queue { - q := &Queue{} +func NewQueue[T any]() *Queue[T] { + q := &Queue[T]{} return q } -func (q *Queue) Len() int { +func (q *Queue[T]) Len() int { q.RLock() defer q.RUnlock() return q.count } -func (q *Queue) Enqueue(item interface{}) { +func (q *Queue[T]) Enqueue(item T) { q.Lock() defer q.Unlock() - n := &node{data: item} + n := &node[T]{data: item} if q.tail == nil { q.tail = n @@ -41,12 +43,12 @@ func (q *Queue) Enqueue(item interface{}) { q.count++ } -func (q *Queue) Dequeue() interface{} { +func (q *Queue[T]) Dequeue() (result T) { q.Lock() defer q.Unlock() if q.head == nil { - return nil + return } n := q.head @@ -59,3 +61,14 @@ func (q *Queue) Dequeue() interface{} { return n.data } + +func (q *Queue[T]) Peek() (result T) { + q.RLock() + defer q.RUnlock() + + if q.head == nil { + return + } + + return q.head.data +} diff --git a/weed/util/queue_test.go b/weed/util/queue_test.go new file mode 100644 index 000000000..a79552893 --- /dev/null +++ b/weed/util/queue_test.go @@ -0,0 +1,22 @@ +package util + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestNewQueue(t *testing.T) { + + q := NewQueue[int]() + + for i := 0; i < 10; i++ { + q.Enqueue(i) + } + + assert.Equal(t, q.Len(), 10) + + for i := 0; i < 10; i++ { + assert.Equal(t, q.Dequeue(), i) + } + +} diff --git a/weed/weed.go b/weed/weed.go index a821cd72f..5139dd39c 100644 --- a/weed/weed.go +++ b/weed/weed.go @@ -20,6 +20,7 @@ import ( "github.com/getsentry/sentry-go" "github.com/seaweedfs/seaweedfs/weed/command" "github.com/seaweedfs/seaweedfs/weed/glog" + util_http "github.com/seaweedfs/seaweedfs/weed/util/http" ) var IsDebug *bool @@ -86,6 +87,7 @@ func main() { return } + util_http.InitGlobalHttpClient() for _, cmd := range commands { if cmd.Name() == args[0] && cmd.Run != nil { cmd.Flag.Usage = func() { cmd.Usage() } |
