aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.github/workflows/container_release.yml58
-rw-r--r--docker/Dockerfile.rocksdb_large59
-rw-r--r--docker/Makefile6
-rw-r--r--go.mod24
-rw-r--r--go.sum4
-rw-r--r--weed/filer/rocksdb/rocksdb_store.go2
-rw-r--r--weed/filer/rocksdb/rocksdb_ttl.go7
-rw-r--r--weed/storage/backend/s3_backend/s3_download.go2
8 files changed, 122 insertions, 40 deletions
diff --git a/.github/workflows/container_release.yml b/.github/workflows/container_release.yml
index b345b1b86..f008f0e70 100644
--- a/.github/workflows/container_release.yml
+++ b/.github/workflows/container_release.yml
@@ -46,14 +46,6 @@ jobs:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
- name: Login to GHCR
- if: github.event_name != 'pull_request'
- uses: docker/login-action@v1
- with:
- registry: ghcr.io
- username: ${{ secrets.GHCR_USERNAME }}
- password: ${{ secrets.GHCR_TOKEN }}
- -
name: Build
uses: docker/build-push-action@v2
with:
@@ -103,20 +95,58 @@ jobs:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
- name: Login to GHCR
+ name: Build
+ uses: docker/build-push-action@v2
+ with:
+ context: ./docker
+ push: ${{ github.event_name != 'pull_request' }}
+ file: ./docker/Dockerfile.go_build_large
+ platforms: linux/amd64, linux/arm, linux/arm64, linux/386
+ tags: ${{ steps.docker_meta.outputs.tags }}
+ labels: ${{ steps.docker_meta.outputs.labels }}
+
+ build-large-release-container_rocksdb:
+ runs-on: [ubuntu-latest]
+
+ steps:
+ -
+ name: Checkout
+ uses: actions/checkout@v2
+ -
+ name: Docker meta
+ id: docker_meta
+ uses: docker/metadata-action@v3
+ with:
+ images: |
+ chrislusf/seaweedfs
+ tags: |
+ type=ref,event=tag,suffix=_large_disk_rocksdb
+ flavor: |
+ latest=false
+ labels: |
+ org.opencontainers.image.title=seaweedfs
+ org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
+ org.opencontainers.image.vendor=Chris Lu
+ -
+ name: Set up QEMU
+ uses: docker/setup-qemu-action@v1
+ -
+ name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v1
+ -
+ name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v1
with:
- registry: ghcr.io
- username: ${{ secrets.GHCR_USERNAME }}
- password: ${{ secrets.GHCR_TOKEN }}
+ username: ${{ secrets.DOCKER_USERNAME }}
+ password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Build
uses: docker/build-push-action@v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
- file: ./docker/Dockerfile.go_build_large
- platforms: linux/amd64, linux/arm, linux/arm64, linux/386
+ file: ./docker/Dockerfile.rocksdb_large
+ platforms: linux/amd64
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
diff --git a/docker/Dockerfile.rocksdb_large b/docker/Dockerfile.rocksdb_large
new file mode 100644
index 000000000..f3bc7eb7d
--- /dev/null
+++ b/docker/Dockerfile.rocksdb_large
@@ -0,0 +1,59 @@
+FROM golang:1.17-buster as builder
+
+RUN apt-get update
+RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
+
+ENV ROCKSDB_VERSION v6.22.1
+
+# build RocksDB
+RUN cd /tmp && \
+ git clone https://github.com/facebook/rocksdb.git /tmp/rocksdb --depth 1 --single-branch --branch $ROCKSDB_VERSION && \
+ cd rocksdb && \
+ make static_lib && \
+ make install-static
+
+ENV CGO_CFLAGS "-I/tmp/rocksdb/include"
+ENV CGO_LDFLAGS "-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd"
+
+# build SeaweedFS
+RUN mkdir -p /go/src/github.com/chrislusf/
+RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
+ARG BRANCH=${BRANCH:-master}
+RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
+RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
+ && export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
+ && go install -tags "5BytesOffset rocksdb" -ldflags "-extldflags -static ${LDFLAGS}"
+
+
+FROM alpine AS final
+LABEL author="Chris Lu"
+COPY --from=builder /go/bin/weed /usr/bin/
+RUN mkdir -p /etc/seaweedfs
+COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
+COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
+RUN apk add fuse snappy gflags
+
+# volume server gprc port
+EXPOSE 18080
+# volume server http port
+EXPOSE 8080
+# filer server gprc port
+EXPOSE 18888
+# filer server http port
+EXPOSE 8888
+# master server shared gprc port
+EXPOSE 19333
+# master server shared http port
+EXPOSE 9333
+# s3 server http port
+EXPOSE 8333
+# webdav server http port
+EXPOSE 7333
+
+RUN mkdir -p /data/filerldb2
+
+VOLUME /data
+
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]
diff --git a/docker/Makefile b/docker/Makefile
index 8efc0ded2..b5fff1c16 100644
--- a/docker/Makefile
+++ b/docker/Makefile
@@ -13,6 +13,12 @@ build: binary
docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local .
rm ./weed
+build_gorocksdb:
+ docker build --no-cache -t chrislusf/gorocksdb -f Dockerfile.go_rocksdb .
+
+build_rocksdb:
+ docker build --no-cache -t chrislusf/seaweedfs:rocksdb -f Dockerfile.rocksdb_large .
+
s3tests_build:
docker build --no-cache -t chrislusf/ceph-s3-tests:local -f Dockerfile.s3tests .
diff --git a/go.mod b/go.mod
index 8c7e88bd7..3b0582c52 100644
--- a/go.mod
+++ b/go.mod
@@ -111,7 +111,6 @@ require (
github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71
github.com/stretchr/testify v1.7.0
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
- github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c
github.com/tidwall/gjson v1.8.1
github.com/tidwall/match v1.0.3
github.com/tidwall/pretty v1.1.0 // indirect
@@ -165,47 +164,26 @@ require (
)
require (
- github.com/apache/thrift v0.14.0 // indirect
github.com/coreos/etcd v3.3.10+incompatible // indirect
- github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
- github.com/d4l3k/messagediff v1.2.1 // indirect
- github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
- github.com/gin-contrib/sse v0.1.0 // indirect
- github.com/gin-gonic/gin v1.6.0 // indirect
- github.com/go-playground/locales v0.13.0 // indirect
- github.com/go-playground/universal-translator v0.17.0 // indirect
- github.com/go-playground/validator/v10 v10.2.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
github.com/jcmturner/rpc/v2 v2.0.2 // indirect
github.com/leodido/go-urn v1.2.0 // indirect
+ github.com/linxGnu/grocksdb v1.6.38 // indirect
github.com/mattn/go-runewidth v0.0.7 // indirect
github.com/miekg/dns v1.1.25-0.20191211073109-8ebf2e419df7 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
- github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3 // indirect
- github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd // indirect
- github.com/pingcap/kvproto v0.0.0-20210806074406-317f69fb54b4 // indirect
- github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4 // indirect
- github.com/pingcap/parser v0.0.0-20210525032559-c37778aff307 // indirect
- github.com/satori/go.uuid v1.2.1-0.20181016170032-d91630c85102 // indirect
github.com/spf13/pflag v1.0.5 // indirect
- github.com/tikv/pd v1.1.0-beta.0.20210323121136-78679e5e209d // indirect
- github.com/twmb/murmur3 v1.1.3 // indirect
- github.com/ugorji/go/codec v1.1.7 // indirect
go.etcd.io/etcd/api/v3 v3.5.0 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.0 // indirect
go.etcd.io/etcd/client/v3 v3.5.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.7.0 // indirect
go.uber.org/zap v1.17.0 // indirect
- golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
- golang.org/x/mod v0.4.2 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
- gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
- gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
)
// replace github.com/seaweedfs/fuse => /Users/chris/go/src/github.com/seaweedfs/fuse
diff --git a/go.sum b/go.sum
index a8b8bbc37..abd342f0c 100644
--- a/go.sum
+++ b/go.sum
@@ -299,6 +299,7 @@ github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/
github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do=
github.com/gin-gonic/gin v1.6.0 h1:Lb3veSYoGaNck69fV2+Vf2juLSsHpMTf3Vk5+X+EDJg=
github.com/gin-gonic/gin v1.6.0/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
+github.com/gin-gonic/gin v1.7.0/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY=
github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
github.com/go-echarts/go-echarts v1.0.0/go.mod h1:qbmyAb/Rl1f2w7wKba1D4LoNq4U164yO4/wedFbcWyo=
github.com/go-errors/errors v1.1.1 h1:ljK/pL5ltg3qoN+OtN6yCv9HWSfMwxSx90GJCZQxYNg=
@@ -340,6 +341,7 @@ github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD87
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY=
github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
+github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
github.com/go-redis/redis/v8 v8.4.4 h1:fGqgxCTR1sydaKI00oQf3OmkU/DIe/I/fYXvGklCIuc=
github.com/go-redis/redis/v8 v8.4.4/go.mod h1:nA0bQuF0i5JFx4Ta9RZxGKXFrQ8cRWntra97f0196iY=
github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
@@ -653,6 +655,8 @@ github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E=
github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
+github.com/linxGnu/grocksdb v1.6.38 h1:2VTvvGFJr5Tkei/Rqp4Sc1J7T65p6reFjBNCcyYNFV8=
+github.com/linxGnu/grocksdb v1.6.38/go.mod h1:/+iSQrn7Izt6kFhHBQvcE6FkklsKXa8hc35pFyFDrDw=
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
diff --git a/weed/filer/rocksdb/rocksdb_store.go b/weed/filer/rocksdb/rocksdb_store.go
index 729da7d9b..f48c3988c 100644
--- a/weed/filer/rocksdb/rocksdb_store.go
+++ b/weed/filer/rocksdb/rocksdb_store.go
@@ -10,7 +10,7 @@ import (
"io"
"os"
- "github.com/tecbot/gorocksdb"
+ gorocksdb "github.com/linxGnu/grocksdb"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
diff --git a/weed/filer/rocksdb/rocksdb_ttl.go b/weed/filer/rocksdb/rocksdb_ttl.go
index faed22310..7e9643083 100644
--- a/weed/filer/rocksdb/rocksdb_ttl.go
+++ b/weed/filer/rocksdb/rocksdb_ttl.go
@@ -5,7 +5,7 @@ package rocksdb
import (
"time"
- "github.com/tecbot/gorocksdb"
+ gorocksdb "github.com/linxGnu/grocksdb"
"github.com/chrislusf/seaweedfs/weed/filer"
)
@@ -38,3 +38,8 @@ func (t *TTLFilter) Filter(level int, key, val []byte) (remove bool, newVal []by
func (t *TTLFilter) Name() string {
return "TTLFilter"
}
+func (t *TTLFilter) SetIgnoreSnapshots(value bool) {
+}
+
+func (t *TTLFilter) Destroy() {
+}
diff --git a/weed/storage/backend/s3_backend/s3_download.go b/weed/storage/backend/s3_backend/s3_download.go
index dbc28446a..224cbbc56 100644
--- a/weed/storage/backend/s3_backend/s3_download.go
+++ b/weed/storage/backend/s3_backend/s3_download.go
@@ -47,7 +47,7 @@ func downloadFromS3(sess s3iface.S3API, destFileName string, sourceBucket string
Key: aws.String(sourceKey),
})
if err != nil {
- return fileSize, fmt.Errorf("failed to download file %s: %v", destFileName, err)
+ return fileSize, fmt.Errorf("failed to download /buckets/%s%s to %s: %v", sourceBucket, sourceKey, destFileName, err)
}
glog.V(1).Infof("downloaded file %s\n", destFileName)
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-21 23:18:35 +0000