blob: 3ccbbe177ef6a5adc52cc2f92d3e3299a258af4d (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
|
name: "S3 PyArrow Parquet Tests"
on:
push:
branches: [master]
paths:
- 'weed/s3api/**'
- 'weed/filer/**'
- 'test/s3/parquet/**'
- '.github/workflows/s3-parquet-tests.yml'
pull_request:
branches: [master]
paths:
- 'weed/s3api/**'
- 'weed/filer/**'
- 'test/s3/parquet/**'
- '.github/workflows/s3-parquet-tests.yml'
workflow_dispatch:
env:
S3_ACCESS_KEY: some_access_key1
S3_SECRET_KEY: some_secret_key1
S3_ENDPOINT_URL: http://localhost:8333
BUCKET_NAME: test-parquet-bucket
jobs:
parquet-integration-tests:
name: PyArrow Parquet Tests (Python ${{ matrix.python-version }})
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
python-version: ['3.9', '3.11', '3.12']
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: ^1.24
cache: true
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
cache-dependency-path: 'test/s3/parquet/requirements.txt'
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y lsof netcat-openbsd
- name: Build SeaweedFS
run: |
cd weed
go build -v
sudo cp weed /usr/local/bin/
weed version
- name: Run PyArrow Parquet integration tests
run: |
cd test/s3/parquet
make test-with-server
env:
SEAWEEDFS_BINARY: weed
S3_PORT: 8333
FILER_PORT: 8888
VOLUME_PORT: 8080
MASTER_PORT: 9333
VOLUME_MAX_SIZE_MB: 50
- name: Run implicit directory fix tests
run: |
cd test/s3/parquet
make test-implicit-dir-with-server
env:
SEAWEEDFS_BINARY: weed
S3_PORT: 8333
FILER_PORT: 8888
VOLUME_PORT: 8080
MASTER_PORT: 9333
- name: Run PyArrow native S3 filesystem tests
run: |
cd test/s3/parquet
make test-native-s3-with-server
env:
SEAWEEDFS_BINARY: weed
S3_PORT: 8333
FILER_PORT: 8888
VOLUME_PORT: 8080
MASTER_PORT: 9333
- name: Run cross-filesystem compatibility tests
run: |
cd test/s3/parquet
TEST_QUICK=1 make test-cross-fs-with-server
env:
SEAWEEDFS_BINARY: weed
S3_PORT: 8333
FILER_PORT: 8888
VOLUME_PORT: 8080
MASTER_PORT: 9333
- name: Run SSE-S3 encryption compatibility tests
run: |
cd test/s3/parquet
make test-sse-s3-compat
env:
SEAWEEDFS_BINARY: weed
S3_PORT: 8333
FILER_PORT: 8888
VOLUME_PORT: 8080
MASTER_PORT: 9333
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v4
with:
name: test-logs-python-${{ matrix.python-version }}
path: |
/tmp/seaweedfs-parquet-*.log
test/s3/parquet/*.log
retention-days: 7
- name: Cleanup
if: always()
run: |
cd test/s3/parquet
make stop-seaweedfs-safe || true
make clean || true
unit-tests:
name: Go Unit Tests (Implicit Directory)
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: ^1.24
cache: true
- name: Run Go unit tests
run: |
cd weed/s3api
go test -v -run TestImplicitDirectory
- name: Run all S3 API tests
run: |
cd weed/s3api
go test -v -timeout 5m
|
