aboutsummaryrefslogtreecommitdiff
path: root/.github
diff options
context:
space:
mode:
Diffstat (limited to '.github')
-rw-r--r--.github/workflows/s3-parquet-tests.yml130
-rw-r--r--.github/workflows/s3-sse-tests.yml12
-rw-r--r--.github/workflows/s3tests.yml2
-rw-r--r--.github/workflows/test-s3-over-https-using-awscli.yml25
4 files changed, 162 insertions, 7 deletions
diff --git a/.github/workflows/s3-parquet-tests.yml b/.github/workflows/s3-parquet-tests.yml
new file mode 100644
index 000000000..8fbd062ef
--- /dev/null
+++ b/.github/workflows/s3-parquet-tests.yml
@@ -0,0 +1,130 @@
+name: "S3 PyArrow Parquet Tests"
+
+on:
+ push:
+ branches: [master]
+ paths:
+ - 'weed/s3api/**'
+ - 'weed/filer/**'
+ - 'test/s3/parquet/**'
+ - '.github/workflows/s3-parquet-tests.yml'
+ pull_request:
+ branches: [master]
+ paths:
+ - 'weed/s3api/**'
+ - 'weed/filer/**'
+ - 'test/s3/parquet/**'
+ - '.github/workflows/s3-parquet-tests.yml'
+ workflow_dispatch:
+
+env:
+ S3_ACCESS_KEY: some_access_key1
+ S3_SECRET_KEY: some_secret_key1
+ S3_ENDPOINT_URL: http://localhost:8333
+ BUCKET_NAME: test-parquet-bucket
+
+jobs:
+ parquet-integration-tests:
+ name: PyArrow Parquet Tests (Python ${{ matrix.python-version }})
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+
+ strategy:
+ fail-fast: false
+ matrix:
+ python-version: ['3.9', '3.11', '3.12']
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Set up Go
+ uses: actions/setup-go@v5
+ with:
+ go-version: ^1.24
+ cache: true
+
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python-version }}
+ cache: 'pip'
+ cache-dependency-path: 'test/s3/parquet/requirements.txt'
+
+ - name: Install system dependencies
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y lsof netcat-openbsd
+
+ - name: Build SeaweedFS
+ run: |
+ cd weed
+ go build -v
+ sudo cp weed /usr/local/bin/
+ weed version
+
+ - name: Run PyArrow Parquet integration tests
+ run: |
+ cd test/s3/parquet
+ make test-with-server
+ env:
+ SEAWEEDFS_BINARY: weed
+ S3_PORT: 8333
+ FILER_PORT: 8888
+ VOLUME_PORT: 8080
+ MASTER_PORT: 9333
+ VOLUME_MAX_SIZE_MB: 50
+
+ - name: Run implicit directory fix tests
+ run: |
+ cd test/s3/parquet
+ make test-implicit-dir-with-server
+ env:
+ SEAWEEDFS_BINARY: weed
+ S3_PORT: 8333
+ FILER_PORT: 8888
+ VOLUME_PORT: 8080
+ MASTER_PORT: 9333
+
+ - name: Upload test logs on failure
+ if: failure()
+ uses: actions/upload-artifact@v4
+ with:
+ name: test-logs-python-${{ matrix.python-version }}
+ path: |
+ /tmp/seaweedfs-parquet-*.log
+ test/s3/parquet/*.log
+ retention-days: 7
+
+ - name: Cleanup
+ if: always()
+ run: |
+ cd test/s3/parquet
+ make stop-seaweedfs-safe || true
+ make clean || true
+
+ unit-tests:
+ name: Go Unit Tests (Implicit Directory)
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Set up Go
+ uses: actions/setup-go@v5
+ with:
+ go-version: ^1.24
+ cache: true
+
+ - name: Run Go unit tests
+ run: |
+ cd weed/s3api
+ go test -v -run TestImplicitDirectory
+
+ - name: Run all S3 API tests
+ run: |
+ cd weed/s3api
+ go test -v -timeout 5m
+
diff --git a/.github/workflows/s3-sse-tests.yml b/.github/workflows/s3-sse-tests.yml
index 5bc9e6be0..42db38d6d 100644
--- a/.github/workflows/s3-sse-tests.yml
+++ b/.github/workflows/s3-sse-tests.yml
@@ -4,6 +4,7 @@ on:
pull_request:
paths:
- 'weed/s3api/s3_sse_*.go'
+ - 'weed/s3api/s3api_object_handlers.go'
- 'weed/s3api/s3api_object_handlers_put.go'
- 'weed/s3api/s3api_object_handlers_copy*.go'
- 'weed/server/filer_server_handlers_*.go'
@@ -14,6 +15,7 @@ on:
branches: [ master, main ]
paths:
- 'weed/s3api/s3_sse_*.go'
+ - 'weed/s3api/s3api_object_handlers.go'
- 'weed/s3api/s3api_object_handlers_put.go'
- 'weed/s3api/s3api_object_handlers_copy*.go'
- 'weed/server/filer_server_handlers_*.go'
@@ -68,11 +70,11 @@ jobs:
# Run tests with automatic server management
# The test-with-server target handles server startup/shutdown automatically
if [ "${{ matrix.test-type }}" = "quick" ]; then
- # Quick tests - basic SSE-C and SSE-KMS functionality
- make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|TestSimpleSSECIntegration"
+ # Quick tests - basic SSE-C and SSE-KMS functionality + Range requests
+ make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior"
else
# Comprehensive tests - SSE-C/KMS functionality, excluding copy operations (pre-existing SSE-C issues)
- make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration"
+ make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior"
fi
- name: Show server logs on failure
@@ -127,8 +129,8 @@ jobs:
uname -a
free -h
- # Run the specific tests that validate AWS S3 SSE compatibility - both SSE-C and SSE-KMS basic functionality
- make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic" || {
+ # Run the specific tests that validate AWS S3 SSE compatibility - both SSE-C and SSE-KMS basic functionality plus Range requests
+ make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|.*RangeRequestsServerBehavior" || {
echo "❌ SSE compatibility test failed, checking logs..."
if [ -f weed-test.log ]; then
echo "=== Server logs ==="
diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml
index 11327c109..c3c6c00d7 100644
--- a/.github/workflows/s3tests.yml
+++ b/.github/workflows/s3tests.yml
@@ -59,7 +59,7 @@ jobs:
# Create clean data directory for this test run
export WEED_DATA_DIR="/tmp/seaweedfs-s3tests-$(date +%s)"
mkdir -p "$WEED_DATA_DIR"
- weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
+ weed -v 3 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
-dir="$WEED_DATA_DIR" \
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
-volume.max=100 -volume.preStopSeconds=1 \
diff --git a/.github/workflows/test-s3-over-https-using-awscli.yml b/.github/workflows/test-s3-over-https-using-awscli.yml
index fd0f8eb4f..9a26f4d82 100644
--- a/.github/workflows/test-s3-over-https-using-awscli.yml
+++ b/.github/workflows/test-s3-over-https-using-awscli.yml
@@ -83,6 +83,29 @@ jobs:
set -e
dd if=/dev/urandom of=generated bs=1M count=32
ETAG=$(aws --no-verify-ssl s3api put-object --bucket bucket --key test-get-obj --body generated | jq -r .ETag)
- aws --no-verify-ssl s3api get-object --bucket bucket --key test-get-obj --if-match ${ETAG:1:32} downloaded
+ # jq -r already removes quotes, so use ETAG directly (handles both simple and multipart ETags)
+ aws --no-verify-ssl s3api get-object --bucket bucket --key test-get-obj --if-match "$ETAG" downloaded
diff -q generated downloaded
rm -f generated downloaded
+
+ - name: Show server logs on failure
+ if: failure()
+ run: |
+ echo "========================================="
+ echo "SeaweedFS Server Logs"
+ echo "========================================="
+ # Note: weed.log is relative to working-directory (weed/)
+ if [ -f weed.log ]; then
+ cat weed.log
+ else
+ echo "No weed.log file found"
+ fi
+
+ - name: Upload server logs on failure
+ if: failure()
+ uses: actions/upload-artifact@v5
+ with:
+ name: seaweedfs-logs
+ # Note: actions don't use defaults.run.working-directory, so path is relative to workspace root
+ path: weed/weed.log
+ retention-days: 3
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-20 20:33:31 +0000