unified builds4.01

7 files changed, 269 insertions, 22 deletions

diff --git a/.github/workflows/container_release1.yml b/.github/workflows/container_release1.yml
index cf74ea3ca..056948fcf 100644
--- a/.github/workflows/container_release1.yml
+++ b/.github/workflows/container_release1.yml
@@ -1,10 +1,13 @@
 name: "docker: build release containers for normal volume"
 
+# DISABLED: Merged into container_release_unified.yml
 on:
-  push:
-    tags:
-      - '*'
-  workflow_dispatch: {}
+  workflow_dispatch:
+    inputs:
+      force_run:
+        description: 'This workflow is disabled. Use container_release_unified.yml instead'
+        required: true
+        default: 'disabled'
 
 permissions:
   contents: read
diff --git a/.github/workflows/container_release2.yml b/.github/workflows/container_release2.yml
index fd14b8f54..f6cd702bc 100644
--- a/.github/workflows/container_release2.yml
+++ b/.github/workflows/container_release2.yml
@@ -1,10 +1,13 @@
 name: "docker: build release containers for large volume"
 
+# DISABLED: Merged into container_release_unified.yml
 on:
-  push:
-    tags:
-      - '*'
-  workflow_dispatch: {}
+  workflow_dispatch:
+    inputs:
+      force_run:
+        description: 'This workflow is disabled. Use container_release_unified.yml instead'
+        required: true
+        default: 'disabled'
 
 permissions:
   contents: read
diff --git a/.github/workflows/container_release3.yml b/.github/workflows/container_release3.yml
index 840737917..e8ff7dcbf 100644
--- a/.github/workflows/container_release3.yml
+++ b/.github/workflows/container_release3.yml
@@ -1,10 +1,13 @@
 name: "docker: build release containers for rocksdb"
 
+# DISABLED: Merged into container_release_unified.yml
 on:
-  push:
-    tags:
-      - '*'
-  workflow_dispatch: {}
+  workflow_dispatch:
+    inputs:
+      force_run:
+        description: 'This workflow is disabled. Use container_release_unified.yml instead'
+        required: true
+        default: 'disabled'
 
 permissions:
   contents: read
diff --git a/.github/workflows/container_release4.yml b/.github/workflows/container_release4.yml
index 8b0c5bfe6..55d658fdf 100644
--- a/.github/workflows/container_release4.yml
+++ b/.github/workflows/container_release4.yml
@@ -1,10 +1,13 @@
 name: "docker: build release containers for all tags"
 
+# DISABLED: Merged into container_release_unified.yml
 on:
-  push:
-    tags:
-      - '*'
-  workflow_dispatch: {}
+  workflow_dispatch:
+    inputs:
+      force_run:
+        description: 'This workflow is disabled. Use container_release_unified.yml instead'
+        required: true
+        default: 'disabled'
 
 permissions:
   contents: read
diff --git a/.github/workflows/container_release5.yml b/.github/workflows/container_release5.yml
index 8f01d8db4..ad3698763 100644
--- a/.github/workflows/container_release5.yml
+++ b/.github/workflows/container_release5.yml
@@ -1,10 +1,13 @@
 name: "docker: build release containers for all tags and large volume"
 
+# DISABLED: Merged into container_release_unified.yml
 on:
-  push:
-    tags:
-      - '*'
-  workflow_dispatch: {}
+  workflow_dispatch:
+    inputs:
+      force_run:
+        description: 'This workflow is disabled. Use container_release_unified.yml instead'
+        required: true
+        default: 'disabled'
 
 permissions:
   contents: read
diff --git a/.github/workflows/container_release_unified.yml b/.github/workflows/container_release_unified.yml
new file mode 100644
index 000000000..3d3a24d6c
--- /dev/null
+++ b/.github/workflows/container_release_unified.yml
@@ -0,0 +1,225 @@
+name: "docker: build all release containers (unified)"
+
+on:
+  push:
+    tags:
+      - '*'
+  workflow_dispatch: {}
+
+permissions:
+  contents: read
+
+# Limit concurrent builds to avoid rate limits
+concurrency:
+  group: release-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      # Build sequentially to avoid rate limits
+      max-parallel: 2
+      matrix:
+        include:
+          # Normal volume - multi-arch
+          - variant: normal
+            platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
+            dockerfile: ./docker/Dockerfile.go_build
+            build_args: ""
+            tag_suffix: ""
+            
+          # Large disk - multi-arch  
+          - variant: large_disk
+            platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
+            dockerfile: ./docker/Dockerfile.go_build
+            build_args: TAGS=5BytesOffset
+            tag_suffix: _large_disk
+            
+          # Full tags - amd64 only
+          - variant: full
+            platforms: linux/amd64
+            dockerfile: ./docker/Dockerfile.go_build
+            build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
+            tag_suffix: _full
+            
+          # Large disk + full tags - amd64 only
+          - variant: large_disk_full
+            platforms: linux/amd64
+            dockerfile: ./docker/Dockerfile.go_build
+            build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
+            tag_suffix: _large_disk_full
+            
+          # RocksDB large disk - amd64 only
+          - variant: rocksdb
+            platforms: linux/amd64
+            dockerfile: ./docker/Dockerfile.rocksdb_large
+            build_args: ""
+            tag_suffix: _large_disk_rocksdb
+            
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        
+      - name: Free Disk Space
+        run: |
+          echo "Available disk space before cleanup:"
+          df -h
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+          sudo apt-get clean
+          sudo rm -rf /var/lib/apt/lists/*
+          sudo docker system prune -af --volumes
+          [ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true
+          [ -d /go/pkg ] && rm -rf /go/pkg || true
+          echo "Available disk space after cleanup:"
+          df -h
+          
+      - name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            chrislusf/seaweedfs
+            ghcr.io/chrislusf/seaweedfs
+          tags: type=ref,event=tag,suffix=${{ matrix.tag_suffix }}
+          flavor: latest=false
+          labels: |
+            org.opencontainers.image.title=seaweedfs
+            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
+            org.opencontainers.image.vendor=Chris Lu
+            
+      - name: Set up QEMU
+        if: contains(matrix.platforms, 'arm')
+        uses: docker/setup-qemu-action@v3
+        
+      - name: Create BuildKit config
+        run: |
+          cat > /tmp/buildkitd.toml <<EOF
+          [registry."docker.io"]
+            mirrors = ["https://mirror.gcr.io"]
+          EOF
+          
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          buildkitd-config: /tmp/buildkitd.toml
+          
+      - name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          
+      - name: Login to GHCR
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ secrets.GHCR_USERNAME }}
+          password: ${{ secrets.GHCR_TOKEN }}
+          
+      - name: Build and push ${{ matrix.variant }}
+        uses: docker/build-push-action@v5
+        env:
+          DOCKER_BUILDKIT: 1
+        with:
+          context: ./docker
+          push: ${{ github.event_name != 'pull_request' }}
+          file: ${{ matrix.dockerfile }}
+          platforms: ${{ matrix.platforms }}
+          # Push to GHCR to avoid Docker Hub rate limits on pulls
+          tags: |
+            ghcr.io/chrislusf/seaweedfs:${{ github.ref_name }}${{ matrix.tag_suffix }}
+          labels: ${{ steps.docker_meta.outputs.labels }}
+          cache-from: type=gha,scope=${{ matrix.variant }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.variant }}
+          build-args: |
+            ${{ matrix.build_args }}
+            BUILDKIT_INLINE_CACHE=1
+            BRANCH=${{ github.sha }}
+            
+      - name: Clean up build artifacts
+        if: always()
+        run: |
+          sudo docker system prune -f
+          sudo rm -rf /tmp/go-build*
+
+  copy-to-dockerhub:
+    runs-on: ubuntu-latest
+    needs: [build]
+    if: github.event_name != 'pull_request'
+    strategy:
+      matrix:
+        variant: [normal, large_disk, full, large_disk_full, rocksdb]
+        include:
+          - variant: normal
+            tag_suffix: ""
+          - variant: large_disk
+            tag_suffix: _large_disk
+          - variant: full
+            tag_suffix: _full
+          - variant: large_disk_full
+            tag_suffix: _large_disk_full
+          - variant: rocksdb
+            tag_suffix: _large_disk_rocksdb
+            
+    steps:
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          
+      - name: Login to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ secrets.GHCR_USERNAME }}
+          password: ${{ secrets.GHCR_TOKEN }}
+          
+      - name: Install crane
+        run: |
+          cd $(mktemp -d)
+          curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
+          sudo mv crane /usr/local/bin/
+          crane version
+          
+      - name: Copy ${{ matrix.variant }} from GHCR to Docker Hub
+        run: |
+          # Function to retry with exponential backoff
+          retry_with_backoff() {
+            local max_attempts=5
+            local timeout=1
+            local attempt=1
+            local exit_code=0
+            
+            while [ $attempt -le $max_attempts ]; do
+              if "$@"; then
+                return 0
+              else
+                exit_code=$?
+              fi
+              
+              if [ $attempt -lt $max_attempts ]; then
+                echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
+                sleep $timeout
+                timeout=$((timeout * 2))
+              fi
+              
+              attempt=$((attempt + 1))
+            done
+            
+            echo "Command failed after $max_attempts attempts" >&2
+            return $exit_code
+          }
+          
+          # Copy multi-arch image from GHCR to Docker Hub with retry
+          # This is much more efficient than pulling/pushing individual arch images
+          echo "Copying ${{ matrix.variant }} from GHCR to Docker Hub..."
+          retry_with_backoff crane copy \
+            ghcr.io/chrislusf/seaweedfs:${{ github.ref_name }}${{ matrix.tag_suffix }} \
+            chrislusf/seaweedfs:${{ github.ref_name }}${{ matrix.tag_suffix }}
+          
+          echo "✓ Successfully copied ${{ matrix.variant }} to Docker Hub"
+
diff --git a/docker/Dockerfile.go_build b/docker/Dockerfile.go_build
index 47b0a28b3..2d9fe99ce 100644
--- a/docker/Dockerfile.go_build
+++ b/docker/Dockerfile.go_build
@@ -1,10 +1,17 @@
 FROM golang:1.24-alpine as builder
 RUN apk add git g++ fuse
 RUN mkdir -p /go/src/github.com/seaweedfs/
-RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
 ARG BRANCH=${BRANCH:-master}
+# Clone with full history and all tags to ensure all commits are available
+RUN git clone --no-single-branch --tags https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
 ARG TAGS
-RUN cd /go/src/github.com/seaweedfs/seaweedfs && (git checkout $BRANCH || (git fetch --unshallow && git checkout $BRANCH))
+RUN cd /go/src/github.com/seaweedfs/seaweedfs && \
+    (git checkout $BRANCH || \
+     (echo "Checkout failed, fetching all history..." && \
+      git fetch --all --tags --prune && \
+      git checkout $BRANCH) || \
+     (echo "ERROR: Branch/commit $BRANCH not found in repository" && \
+      echo "Available branches:" && git branch -a && exit 1))
 RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
   && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
   && CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"