aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.github/workflows/release.yml6
-rw-r--r--.github/workflows/release_binaries.yml4
-rw-r--r--.travis.yml.disabled (renamed from .travis.yml)4
-rw-r--r--Makefile3
-rw-r--r--README.md9
-rw-r--r--docker/Dockerfile2
-rw-r--r--docker/Dockerfile.gccgo_build43
-rw-r--r--docker/Dockerfile.go_build2
-rw-r--r--docker/Dockerfile.go_build_large2
-rw-r--r--go.mod95
-rw-r--r--go.sum26
-rw-r--r--weed/command/filer_copy.go59
-rw-r--r--weed/server/common.go15
-rw-r--r--weed/server/filer_server_handlers_read.go3
-rw-r--r--weed/shell/command_remote_configure.go2
-rw-r--r--weed/storage/needle/crc.go2
16 files changed, 240 insertions, 37 deletions
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 828396230..7905b9420 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -1,4 +1,4 @@
-name: Release
+name: Build Dev Binaries
on:
push:
@@ -35,7 +35,7 @@ jobs:
- name: Go Release Binaries
uses: wangyoucao577/go-release-action@v1.17
with:
- goversion: 1.16
+ goversion: 1.17
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
@@ -52,7 +52,7 @@ jobs:
- name: Go Release Binaries
uses: wangyoucao577/go-release-action@v1.17
with:
- goversion: 1.16
+ goversion: 1.17
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
diff --git a/.github/workflows/release_binaries.yml b/.github/workflows/release_binaries.yml
index 8acb983fb..ca3457c8b 100644
--- a/.github/workflows/release_binaries.yml
+++ b/.github/workflows/release_binaries.yml
@@ -30,7 +30,7 @@ jobs:
- name: Go Release Binaries
uses: wangyoucao577/go-release-action@v1.19
with:
- goversion: 1.16
+ goversion: 1.17
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
@@ -45,7 +45,7 @@ jobs:
- name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@v1.19
with:
- goversion: 1.16
+ goversion: 1.17
github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }}
goarch: ${{ matrix.goarch }}
diff --git a/.travis.yml b/.travis.yml.disabled
index 7934e1ead..d34427fc7 100644
--- a/.travis.yml
+++ b/.travis.yml.disabled
@@ -1,7 +1,7 @@
sudo: false
language: go
go:
- - 1.16.x
+ - 1.17.x
before_install:
- export PATH=/home/travis/gopath/bin:$PATH
@@ -43,4 +43,4 @@ deploy:
on:
tags: true
repo: chrislusf/seaweedfs
- go: 1.16.x
+ go: 1.17.x
diff --git a/Makefile b/Makefile
index 9a62eb9fe..b73310dcf 100644
--- a/Makefile
+++ b/Makefile
@@ -36,6 +36,9 @@ deps:
build: deps
go build $(GO_FLAGS) -ldflags "$(LDFLAGS)" -o $(BINARY) $(SOURCE_DIR)
+gccgo_build: deps
+ go build $(GO_FLAGS) -ldflags "$(LDFLAGS)" -compiler=gccgo -tags gccgo,noasm -o $(BINARY) $(SOURCE_DIR)
+
install: deps
go install $(GO_FLAGS) -ldflags "$(LDFLAGS)" $(SOURCE_DIR)
diff --git a/README.md b/README.md
index 865cc8c99..842b4211c 100644
--- a/README.md
+++ b/README.md
@@ -3,7 +3,7 @@
[![Slack](https://img.shields.io/badge/slack-purple)](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTEyYzZmZWYzOGQ3MDJlZWMzYmI0OTE4OTJiZjJjODBmMzUxNmYwODg0YjY3MTNlMjBmZDQ1NzQ5NDJhZWI2ZmY)
[![Twitter](https://img.shields.io/twitter/follow/seaweedfs.svg?style=social&label=Follow)](https://twitter.com/intent/follow?screen_name=seaweedfs)
-[![Build Status](https://travis-ci.com/chrislusf/seaweedfs.svg?branch=master)](https://travis-ci.com/chrislusf/seaweedfs)
+[![Build Status](https://img.shields.io/github/workflow/status/chrislusf/seaweedfs/Go)](https://github.com/chrislusf/seaweedfs/actions/workflows/go.yml)
[![GoDoc](https://godoc.org/github.com/chrislusf/seaweedfs/weed?status.svg)](https://godoc.org/github.com/chrislusf/seaweedfs/weed)
[![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/chrislusf/seaweedfs/wiki)
[![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs/)
@@ -169,7 +169,7 @@ Faster and Cheaper than direct cloud storage!
[ActiveActiveAsyncReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Active-Active-cross-cluster-continuous-synchronization
[FilerStoreReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Store-Replication
[KeyLargeValueStore]: https://github.com/chrislusf/seaweedfs/wiki/Filer-as-a-Key-Large-Value-Store
-[RemoteStorage]: https://github.com/chrislusf/seaweedfs/wiki/Remote-Storage-Architecture
+[RemoteStorage]: https://github.com/chrislusf/seaweedfs/wiki/Cloud-Cache-Architecture
[Back to TOC](#table-of-contents)
@@ -590,6 +590,7 @@ The text of this page is available for modification and reuse under the terms of
[Back to TOC](#table-of-contents)
-## Stargazers over time ##
+## Stargazers over time
+
+[![Stargazers over time](https://starchart.cc/chrislusf/seaweedfs.svg)](https://starchart.cc/chrislusf/seaweedfs)
-[![Stargazers over time](https://starcharts.herokuapp.com/chrislusf/seaweedfs.svg)](https://starcharts.herokuapp.com/chrislusf/seaweedfs)
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 2165466ca..cf8eac251 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -16,7 +16,7 @@ RUN \
# Install SeaweedFS and Supercronic ( for cron job mode )
apk add --no-cache --virtual build-dependencies --update wget curl ca-certificates && \
apk add fuse && \
- wget -P /tmp https://github.com/$(curl -s -L https://github.com/chrislusf/seaweedfs/releases/${RELEASE} | egrep -o "chrislusf/seaweedfs/releases/download/.*/linux_$ARCH.tar.gz") && \
+ wget -P /tmp https://github.com/$(curl -s -L https://github.com/chrislusf/seaweedfs/releases/${RELEASE} | egrep -o "chrislusf/seaweedfs/releases/download/.*/linux_$ARCH.tar.gz" | head -n 1) && \
tar -C /usr/bin/ -xzvf /tmp/linux_$ARCH.tar.gz && \
curl -fsSLO "$SUPERCRONIC_URL" && \
echo "${SUPERCRONIC_SHA1SUM} ${SUPERCRONIC}" | sha1sum -c - && \
diff --git a/docker/Dockerfile.gccgo_build b/docker/Dockerfile.gccgo_build
new file mode 100644
index 000000000..e42a593cb
--- /dev/null
+++ b/docker/Dockerfile.gccgo_build
@@ -0,0 +1,43 @@
+FROM gcc:11 as builder
+RUN mkdir -p /go/src/github.com/chrislusf/
+RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
+ARG BRANCH=${BRANCH:-master}
+RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
+RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
+ && apt-get update \
+ && apt-get install -y golang-src \
+ && export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
+ && CGO_ENABLED=0 go install -ldflags "-extldflags -static ${LDFLAGS}" -compiler=gccgo -tags gccgo,noasm
+
+FROM alpine AS final
+LABEL author="Chris Lu"
+COPY --from=builder /go/bin/weed /usr/bin/
+RUN mkdir -p /etc/seaweedfs
+COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
+COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
+RUN apk add fuse # for weed mount
+
+# volume server gprc port
+EXPOSE 18080
+# volume server http port
+EXPOSE 8080
+# filer server gprc port
+EXPOSE 18888
+# filer server http port
+EXPOSE 8888
+# master server shared gprc port
+EXPOSE 19333
+# master server shared http port
+EXPOSE 9333
+# s3 server http port
+EXPOSE 8333
+# webdav server http port
+EXPOSE 7333
+
+RUN mkdir -p /data/filerldb2
+
+VOLUME /data
+
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file
diff --git a/docker/Dockerfile.go_build b/docker/Dockerfile.go_build
index 01334280d..3a62cf5a3 100644
--- a/docker/Dockerfile.go_build
+++ b/docker/Dockerfile.go_build
@@ -1,4 +1,4 @@
-FROM amd64/golang:1.16-alpine as builder
+FROM amd64/golang:1.17-alpine as builder
RUN apk add git g++ fuse
RUN mkdir -p /go/src/github.com/chrislusf/
RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
diff --git a/docker/Dockerfile.go_build_large b/docker/Dockerfile.go_build_large
index 2d67a1924..b0433257c 100644
--- a/docker/Dockerfile.go_build_large
+++ b/docker/Dockerfile.go_build_large
@@ -1,4 +1,4 @@
-FROM amd64/golang:1.16-alpine as builder
+FROM amd64/golang:1.17-alpine as builder
RUN apk add git g++ fuse
RUN mkdir -p /go/src/github.com/chrislusf/
RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
diff --git a/go.mod b/go.mod
index 62df38f1d..6d35cd7b3 100644
--- a/go.mod
+++ b/go.mod
@@ -1,22 +1,33 @@
module github.com/chrislusf/seaweedfs
-go 1.16
+go 1.17
require (
+ cloud.google.com/go v0.58.0 // indirect
cloud.google.com/go/pubsub v1.3.1
cloud.google.com/go/storage v1.9.0
+ github.com/Azure/azure-pipeline-go v0.2.2 // indirect
github.com/Azure/azure-storage-blob-go v0.9.0
+ github.com/BurntSushi/toml v0.3.1 // indirect
+ github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 // indirect
github.com/OneOfOne/xxhash v1.2.2
github.com/Shopify/sarama v1.23.1
github.com/aws/aws-sdk-go v1.34.30
+ github.com/beorn7/perks v1.0.1 // indirect
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72
github.com/bwmarrin/snowflake v0.3.0
github.com/cespare/xxhash v1.1.0
+ github.com/cespare/xxhash/v2 v2.1.1 // indirect
github.com/chrislusf/raft v1.0.7
github.com/coreos/go-semver v0.3.0 // indirect
+ github.com/coreos/go-systemd/v22 v22.0.0 // indirect
+ github.com/davecgh/go-spew v1.1.1 // indirect
+ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/disintegration/imaging v1.6.2
github.com/dustin/go-humanize v1.0.0
github.com/eapache/go-resiliency v1.2.0 // indirect
+ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
+ github.com/eapache/queue v1.1.0 // indirect
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a
github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
@@ -24,9 +35,12 @@ require (
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
github.com/fclairamb/ftpserverlib v0.8.0
github.com/frankban/quicktest v1.7.2 // indirect
+ github.com/fsnotify/fsnotify v1.4.9 // indirect
github.com/go-errors/errors v1.1.1 // indirect
github.com/go-redis/redis/v8 v8.4.4
github.com/go-sql-driver/mysql v1.5.0
+ github.com/go-stack/stack v1.8.0 // indirect
+ github.com/go-zookeeper/zk v1.0.2 // indirect
github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d
github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 // indirect
github.com/golang-jwt/jwt v3.2.1+incompatible
@@ -34,35 +48,69 @@ require (
github.com/golang/protobuf v1.4.3
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.0.0
+ github.com/google/go-cmp v0.5.5 // indirect
github.com/google/uuid v1.1.1
+ github.com/google/wire v0.4.0 // indirect
+ github.com/googleapis/gax-go v2.0.2+incompatible // indirect
+ github.com/googleapis/gax-go/v2 v2.0.5 // indirect
github.com/gorilla/mux v1.7.4
github.com/gorilla/websocket v1.4.1 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4
github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect
+ github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
+ github.com/hashicorp/errwrap v1.0.0 // indirect
+ github.com/hashicorp/go-multierror v1.0.0 // indirect
+ github.com/hashicorp/go-uuid v1.0.1 // indirect
+ github.com/hashicorp/hcl v1.0.0 // indirect
github.com/jcmturner/gofork v1.0.0 // indirect
github.com/jinzhu/copier v0.2.8
+ github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/json-iterator/go v1.1.11
+ github.com/jstemmer/go-junit-report v0.9.1 // indirect
github.com/karlseguin/ccache/v2 v2.0.7
+ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/klauspost/compress v1.10.9 // indirect
github.com/klauspost/cpuid v1.2.1 // indirect
- github.com/klauspost/crc32 v1.2.0
github.com/klauspost/reedsolomon v1.9.2
+ github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
github.com/kurin/blazer v0.5.3
github.com/lib/pq v1.10.0
github.com/magiconair/properties v1.8.1 // indirect
+ github.com/mailru/easyjson v0.7.1 // indirect
+ github.com/mattn/go-ieproxy v0.0.1 // indirect
+ github.com/mattn/go-isatty v0.0.12 // indirect
github.com/mattn/go-runewidth v0.0.4 // indirect
+ github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+ github.com/mitchellh/mapstructure v1.1.2 // indirect
+ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+ github.com/modern-go/reflect2 v1.0.1 // indirect
+ github.com/nats-io/jwt v1.0.1 // indirect
+ github.com/nats-io/nats.go v1.10.0 // indirect
+ github.com/nats-io/nkeys v0.2.0 // indirect
+ github.com/nats-io/nuid v1.0.1 // indirect
github.com/olivere/elastic/v7 v7.0.19
+ github.com/pelletier/go-toml v1.7.0 // indirect
github.com/peterh/liner v1.1.0
github.com/pierrec/lz4 v2.2.7+incompatible // indirect
+ github.com/pkg/errors v0.9.1 // indirect
+ github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/posener/complete v1.2.3
github.com/pquerna/cachecontrol v0.1.0
github.com/prometheus/client_golang v1.11.0
+ github.com/prometheus/client_model v0.2.0 // indirect
+ github.com/prometheus/common v0.26.0 // indirect
+ github.com/prometheus/procfs v0.6.0 // indirect
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect
- github.com/seaweedfs/fuse v1.1.8
+ github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
+ github.com/seaweedfs/fuse v1.2.0
github.com/seaweedfs/goexif v1.0.2
+ github.com/sirupsen/logrus v1.6.0 // indirect
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
github.com/spaolacci/murmur3 v1.1.0 // indirect
+ github.com/spf13/afero v1.6.0 // indirect
+ github.com/spf13/cast v1.3.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
+ github.com/spf13/pflag v1.0.3 // indirect
github.com/spf13/viper v1.4.0
github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71
github.com/stretchr/testify v1.6.1
@@ -70,6 +118,7 @@ require (
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c
github.com/tidwall/gjson v1.8.1
github.com/tidwall/match v1.0.3
+ github.com/tidwall/pretty v1.1.0 // indirect
github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
github.com/valyala/bytebufferpool v1.0.0
github.com/viant/assertly v0.5.4 // indirect
@@ -77,21 +126,55 @@ require (
github.com/viant/toolbox v0.33.2 // indirect
github.com/willf/bitset v1.1.10 // indirect
github.com/willf/bloom v2.0.3+incompatible
+ github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+ github.com/xdg-go/scram v1.0.2 // indirect
+ github.com/xdg-go/stringprep v1.0.2 // indirect
+ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
go.etcd.io/etcd v3.3.15+incompatible
go.mongodb.org/mongo-driver v1.7.0
+ go.opencensus.io v0.22.4 // indirect
+ go.opentelemetry.io/otel v0.15.0 // indirect
+ go.uber.org/atomic v1.6.0 // indirect
+ go.uber.org/multierr v1.5.0 // indirect
+ go.uber.org/zap v1.14.1 // indirect
gocloud.dev v0.20.0
gocloud.dev/pubsub/natspubsub v0.20.0
gocloud.dev/pubsub/rabbitpubsub v0.20.0
- golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect
- golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb
- golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40
+ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect
+ golang.org/x/image v0.0.0-20200119044424-58c23975cae1
+ golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
+ golang.org/x/mod v0.3.0 // indirect
+ golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
+ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
+ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a // indirect
+ golang.org/x/sys v0.0.0-20210817142637-7d9622a276b7
+ golang.org/x/text v0.3.6 // indirect
golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78
+ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/api v0.26.0
+ google.golang.org/appengine v1.6.6 // indirect
+ google.golang.org/genproto v0.0.0-20200608115520-7c474a2e3482 // indirect
google.golang.org/grpc v1.29.1
google.golang.org/protobuf v1.26.0-rc.1
+ gopkg.in/inf.v0 v0.9.1 // indirect
+ gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect
+ gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect
gopkg.in/jcmturner/goidentity.v3 v3.0.0 // indirect
gopkg.in/jcmturner/gokrb5.v7 v7.3.0 // indirect
+ gopkg.in/jcmturner/rpc.v1 v1.1.0 // indirect
+ gopkg.in/yaml.v2 v2.3.0 // indirect
+ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
+ honnef.co/go/tools v0.0.1-2020.1.4 // indirect
+ modernc.org/b v1.0.0 // indirect
+ modernc.org/cc/v3 v3.33.5 // indirect
+ modernc.org/ccgo/v3 v3.9.4 // indirect
+ modernc.org/libc v1.9.5 // indirect
+ modernc.org/mathutil v1.2.2 // indirect
+ modernc.org/memory v1.0.4 // indirect
+ modernc.org/opt v0.1.1 // indirect
modernc.org/sqlite v1.10.7
+ modernc.org/strutil v1.1.0 // indirect
+ modernc.org/token v1.0.0 // indirect
)
// replace github.com/seaweedfs/fuse => /Users/chris/go/src/github.com/seaweedfs/fuse
diff --git a/go.sum b/go.sum
index 4541a5f7e..32a2ce22d 100644
--- a/go.sum
+++ b/go.sum
@@ -256,8 +256,6 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe
github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
-github.com/gocql/gocql v0.0.0-20190829130954-e163eff7a8c6 h1:P66kRWyEoIx6URKgAC3ijx9jo9gEid7bEhLQ/Z0G65A=
-github.com/gocql/gocql v0.0.0-20190829130954-e163eff7a8c6/go.mod h1:Q7Sru5153KG8D9zwueuQJB3ccJf9/bIwF/x8b3oKgT8=
github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d h1:k544nNVphXK4Yt0FTduvOvCfJabEY/DMkdNw0zpCwBE=
github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8=
github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -298,9 +296,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
@@ -441,8 +437,6 @@ github.com/klauspost/compress v1.10.9 h1:pPRt1Z78crspaHISkpSSHjDlx+Tt9suHe519dsI
github.com/klauspost/compress v1.10.9/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w=
github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
-github.com/klauspost/crc32 v1.2.0 h1:0VuyqOCruD33/lJ/ojXNvzVyl8Zr5zdTmj9l9qLZ86I=
-github.com/klauspost/crc32 v1.2.0/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg=
github.com/klauspost/reedsolomon v1.9.2 h1:E9CMS2Pqbv+C7tsrYad4YC9MfhnMVWhMRsTi7U0UB18=
github.com/klauspost/reedsolomon v1.9.2/go.mod h1:CwCi+NUr9pqSVktrkN+Ondf06rkhYZ/pcNv7fu+8Un4=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -627,8 +621,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
-github.com/seaweedfs/fuse v1.1.8 h1:YFSDPotG4uhQzV7ooDUvQ8BRVy5rM1XCFPJAmAsZz68=
-github.com/seaweedfs/fuse v1.1.8/go.mod h1:+PP6WlkrRUG6KPE+Th2EX5To/PjHaFsvqg/UgQ39aj8=
+github.com/seaweedfs/fuse v1.2.0 h1:m4Ar3I0TY/Tzyb80lfJiXUKFcMBHqA5e/cZwxQtilZo=
+github.com/seaweedfs/fuse v1.2.0/go.mod h1:iwbDQv5BZACY54r6AO/6xsLNuMaYcBKSkLTZVfmK594=
github.com/seaweedfs/goexif v1.0.2 h1:p+rTXYdQ2mgxd+1JaTrQ9N8DvYuw9UH9xgYmJ+Bb29E=
github.com/seaweedfs/goexif v1.0.2/go.mod h1:MrKs5LK0HXdffrdCZrW3OIMegL2xXpC6ThLyXMyjdrk=
github.com/secsy/goftp v0.0.0-20190720192957-f31499d7c79a h1:C6IhVTxNkhlb0tlCB6JfHOUv1f0xHPK7V8X4HlJZEJw=
@@ -655,8 +649,9 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.3.1 h1:GPTpEAuNr98px18yNQ66JllNil98wfRZ/5Ukny8FeQA=
github.com/spf13/afero v1.3.1/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=
+github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY=
+github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
@@ -861,8 +856,9 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/
golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U=
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c=
+golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -920,7 +916,6 @@ golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200317113312-5766fd39f98d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -930,17 +925,22 @@ golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210817142637-7d9622a276b7 h1:lQ8Btl/sJr2+f4ql7ffKUKfnV0BsgsICvm0oEeINAQY=
+golang.org/x/sys v0.0.0-20210817142637-7d9622a276b7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.go
index 722f64679..818ae5f23 100644
--- a/weed/command/filer_copy.go
+++ b/weed/command/filer_copy.go
@@ -531,6 +531,11 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
return uploadError
}
+ manifestedChunks, manifestErr := filer.MaybeManifestize(worker.saveDataAsChunk, chunks)
+ if manifestErr != nil {
+ return fmt.Errorf("create manifest: %v", manifestErr)
+ }
+
if err := pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.CreateEntryRequest{
Directory: task.destinationUrlPath,
@@ -548,7 +553,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
Collection: collection,
TtlSec: worker.options.ttlSec,
},
- Chunks: chunks,
+ Chunks: manifestedChunks,
},
}
@@ -583,3 +588,55 @@ func detectMimeType(f *os.File) string {
}
return mimeType
}
+
+func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, offset int64) (chunk *filer_pb.FileChunk, collection, replication string, err error) {
+
+ var fileId, host string
+ var auth security.EncodedJwt
+
+ if flushErr := pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+
+ ctx := context.Background()
+
+ assignErr := util.Retry("assignVolume", func() error {
+ request := &filer_pb.AssignVolumeRequest{
+ Count: 1,
+ Replication: *worker.options.replication,
+ Collection: *worker.options.collection,
+ TtlSec: worker.options.ttlSec,
+ DiskType: *worker.options.diskType,
+ Path: name,
+ }
+
+ resp, err := client.AssignVolume(ctx, request)
+ if err != nil {
+ return fmt.Errorf("assign volume failure %v: %v", request, err)
+ }
+ if resp.Error != "" {
+ return fmt.Errorf("assign volume failure %v: %v", request, resp.Error)
+ }
+
+ fileId, host, auth = resp.FileId, resp.Url, security.EncodedJwt(resp.Auth)
+ collection, replication = resp.Collection, resp.Replication
+
+ return nil
+ })
+ if assignErr != nil {
+ return assignErr
+ }
+
+ return nil
+ }); flushErr != nil {
+ return nil, collection, replication, fmt.Errorf("filerGrpcAddress assign volume: %v", flushErr)
+ }
+
+ fileUrl := fmt.Sprintf("http://%s/%s", host, fileId)
+ uploadResult, flushErr, _ := operation.Upload(fileUrl, name, worker.options.cipher, reader, false, "", nil, auth)
+ if flushErr != nil {
+ return nil, collection, replication, fmt.Errorf("upload data: %v", flushErr)
+ }
+ if uploadResult.Error != "" {
+ return nil, collection, replication, fmt.Errorf("upload result: %v", uploadResult.Error)
+ }
+ return uploadResult.ToPbFileChunk(fileId, offset), collection, replication, nil
+}
diff --git a/weed/server/common.go b/weed/server/common.go
index 13d20d879..ad3842190 100644
--- a/weed/server/common.go
+++ b/weed/server/common.go
@@ -33,8 +33,21 @@ func init() {
go serverStats.Start()
}
+// bodyAllowedForStatus is a copy of http.bodyAllowedForStatus non-exported function.
+func bodyAllowedForStatus(status int) bool {
+ switch {
+ case status >= 100 && status <= 199:
+ return false
+ case status == http.StatusNoContent:
+ return false
+ case status == http.StatusNotModified:
+ return false
+ }
+ return true
+}
+
func writeJson(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) (err error) {
- if httpStatus == http.StatusNoContent {
+ if !bodyAllowedForStatus(httpStatus) {
return
}
diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go
index fc9cacf39..054a1bd00 100644
--- a/weed/server/filer_server_handlers_read.go
+++ b/weed/server/filer_server_handlers_read.go
@@ -142,6 +142,9 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
if rangeReq := r.Header.Get("Range"); rangeReq == "" {
ext := filepath.Ext(filename)
+ if len(ext) > 0 {
+ ext = strings.ToLower(ext)
+ }
width, height, mode, shouldResize := shouldResizeImages(ext, r)
if shouldResize {
data, err := filer.ReadAll(fs.filer.MasterClient, entry.Chunks)
diff --git a/weed/shell/command_remote_configure.go b/weed/shell/command_remote_configure.go
index 7a9ad1f65..53c7ea65d 100644
--- a/weed/shell/command_remote_configure.go
+++ b/weed/shell/command_remote_configure.go
@@ -95,7 +95,7 @@ func (c *commandRemoteConfigure) listExistingRemoteStorages(commandEnv *CommandE
return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, entry.Name, err)
}
- conf.S3SecretKey = ""
+ conf.S3SecretKey = strings.Repeat("*", len(conf.S3SecretKey))
m := jsonpb.Marshaler{
EmitDefaults: false,
diff --git a/weed/storage/needle/crc.go b/weed/storage/needle/crc.go
index 4476631c2..b7d3edd83 100644
--- a/weed/storage/needle/crc.go
+++ b/weed/storage/needle/crc.go
@@ -4,7 +4,7 @@ import (
"fmt"
"io"
- "github.com/klauspost/crc32"
+ "hash/crc32"
"github.com/chrislusf/seaweedfs/weed/util"
)
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-21 15:51:27 +0000