leftyer/JuiceFS
1
0
Fork 0
mirror of https://gitee.com/juicedata/JuiceFS.git synced 2025-12-06 09:39:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

CI: refactor fstests (#4841)

Browse Source
This commit is contained in:
Zhou Cheng
2024-05-13 15:21:56 +08:00
committed by GitHub
parent b553e1d7e3
commit ec619b3c7f
10 changed files with 265 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

165
.github/workflows/integrationtests.yml vendored
View File

@@ -5,18 +5,20 @@ on:
branches:
- 'main'
- 'release-*'
paths-ignore:
- 'docs/**'
- '**.md'
paths:
- '**.c'
- '**.go'
- 'Makefile'
- 'integrationtests.yml'
pull_request:
#The branches below must be a subset of the branches above
branches:
- 'main'
- 'release-*'
paths-ignore:
- 'docs/**'
- '**.md'
- '.github/**'
paths:
- '**.c'
- '**.go'
- 'Makefile'
- 'integrationtests.yml'
workflow_dispatch:
inputs:
debug:
@@ -26,37 +28,74 @@ on:
default: false
jobs:
integrationtests:
timeout-minutes: 60
build-matrix:
runs-on: ubuntu-20.04
steps:
- id: set-matrix
run: |
echo "github.event_name is ${{github.event_name}}"
echo "GITHUB_REF_NAME is ${GITHUB_REF_NAME}"
if [[ "${{github.event_name}}" == "schedule" || "${{github.event_name}}" == "workflow_dispatch" ]]; then
echo 'meta_matrix=["sqlite3", "redis", "mysql", "tikv", "tidb", "postgres", "badger", "mariadb", "fdb"]' >> $GITHUB_OUTPUT
elif [[ "${{github.event_name}}" == "pull_request" || "${{github.event_name}}" == "push" ]]; then
echo 'meta_matrix=["redis", "mysql"]' >> $GITHUB_OUTPUT
else
echo "event_name is not supported" && exit 1
fi
outputs:
meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }}
integrationtests:
timeout-minutes: 120
runs-on: ubuntu-20.04
needs: build-matrix
strategy:
fail-fast: false
matrix:
meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }}
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 1
- name: Set Variable
id: vars
run: |
if [ "${{matrix.meta}}" == "fdb" ]; then
echo "target=juicefs.fdb" >> $GITHUB_OUTPUT
else
echo "target=juicefs" >> $GITHUB_OUTPUT
fi
- name: Build
timeout-minutes: 10
uses: ./.github/actions/build
with:
target: ${{steps.vars.outputs.target}}
- name: Copy
run: |
cp juicefs /tmp/mount.juicefs
- name: Run Redis
run: |
sudo docker run -d --name redis -v redis-data:/data \
-p 6379:6379 redis redis-server --appendonly yes
- name: Prepare meta db
run: |
chmod +x .github/scripts/start_meta_engine.sh
source .github/scripts/start_meta_engine.sh
start_meta_engine ${{matrix.meta}}
meta_url=$(get_meta_url ${{matrix.meta}})
create_database $meta_url
- name: Juicefs Format
run: |
sudo ./juicefs format redis://127.0.0.1:6379/1 pics
source .github/scripts/start_meta_engine.sh
meta_url=$(get_meta_url ${{matrix.meta}})
sudo ./juicefs format $meta_url --trash-days 0 pics
- name: Juicefs Mount
run: |
sudo ./juicefs mount -d redis://127.0.0.1:6379/1 /jfs/ --enable-xattr --no-usage-report &
source .github/scripts/start_meta_engine.sh
meta_url=$(get_meta_url ${{matrix.meta}})
sudo ./juicefs mount -d $meta_url /jfs --no-usage-report --enable-xattr
stat /jfs/.accesslog
- name: Fslock Test
timeout-minutes: 5
run: |
cd /jfs/
git clone https://github.com/danjacques/gofslock.git
@@ -64,52 +103,74 @@ jobs:
go test -v ./fslock/...
stat /jfs/
- name: Pyxattr Test
- name: flock test
timeout-minutes: 5
run: |
git clone https://github.com/iustin/pyxattr.git
cd pyxattr
pip3 install pytest
pip3 install pyxattr
stat /jfs/
TEST_DIR=/jfs/ python3 -m pytest tests
git clone https://github.com/gofrs/flock.git
mkdir /jfs/tmp
cd flock && go mod init github.com/gofrs/flock.git && go mod tidy && TMPDIR=/jfs/tmp go test .
- name: Fstests
- name: make secfs.test
run: |
sudo .github/scripts/apt_install.sh libacl1-dev attr
sudo DURATION=60 make -C fstests fsx
sudo make -C fstests xattrs
sudo make -C fstests flock
sudo .github/scripts/apt_install.sh libacl1-dev
git clone https://github.com/billziss-gh/secfs.test.git
make -C secfs.test
- name: Fsx Test
timeout-minutes: 15
run: |
if [[ "${{github.event_name}}" == "schedule" || "${{github.event_name}}" == "workflow_dispatch" ]] ; then
duration=900
else
duration=300
fi
sudo touch /jfs/fsx.out
sudo rm -f /tmp/fsx.out
sudo ln -s /jfs/fsx.out /tmp/fsx.out
sudo secfs.test/tools/bin/fsx -d $duration -p 10000 -F 10000000 /tmp/fsx.out
# sudo secfs.test/tools/bin/fsx -d 180 -p 10000 -F 100000 /jfs/fsx.out
- name: Fsracer
#https://github.com/gfx/example-github-actions-with-tty
- name: Fsracer Test
timeout-minutes: 15
shell: 'script -q -e -c "bash {0}"'
run: |
sudo DURATION=60 make -C fstests fsracer
- name: MountJuiceFSTest
run: |
sudo /tmp/mount.juicefs redis://127.0.0.1:6379/1 /tmp/mount-jfs
[ `stat --format "%i" /tmp/mount-jfs` -eq 1 ]
if [[ "${{github.event_name}}" == "schedule" || "${{github.event_name}}" == "workflow_dispatch" ]] ; then
duration=900
else
duration=300
fi
sudo secfs.test/tools/bin/fsracer $duration /jfs
- name: log
if: always()
run: |
tail -300 /var/log/juicefs.log
grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
- name: Setup upterm session
if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
timeout-minutes: 60
uses: lhotari/action-upterm@v1
success-all-test:
runs-on: ubuntu-latest
needs: [integrationtests]
if: always()
steps:
- uses: technote-space/workflow-conclusion-action@v3
- uses: actions/checkout@v3
- name: Check Failure
if: env.WORKFLOW_CONCLUSION == 'failure'
run: exit 1
- name: Send Slack Notification
if: failure()
if: failure() && github.event_name != 'workflow_dispatch'
uses: juicedata/slack-notify-action@main
with:
channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"
- name: Setup upterm session
if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
timeout-minutes: 60
uses: lhotari/action-upterm@v1
- name: Success
if: success()
run: echo "All Done"

2
.github/workflows/mutate-test.yml vendored
View File

@@ -148,7 +148,7 @@ jobs:
docker run -d -p 9000:9000 -p 9001:9001 -e "MINIO_ROOT_USER=testUser" -e "MINIO_ROOT_PASSWORD=testUserPassword" quay.io/minio/minio:RELEASE.2022-01-25T19-56-04Z server /data --console-address ":9001"
go install github.com/minio/mc@RELEASE.2022-01-07T06-01-38Z && mc config host add local http://127.0.0.1:9000 testUser testUserPassword && mc mb local/testbucket
make
sudo make -C fstests setup
# sudo make -C fstests setup
- name: run mutate test
# timeout-minutes: 120

18
.github/workflows/pjdfstest.yml vendored
View File

@@ -8,6 +8,7 @@ on:
paths-ignore:
- 'docs/**'
- '**.md'
- '.github/**'
pull_request:
#The branches below must be a subset of the branches above
branches:
@@ -34,15 +35,12 @@ jobs:
- id: set-matrix
run: |
echo "github.event_name is ${{github.event_name}}"
echo "GITHUB_REF_NAME is ${GITHUB_REF_NAME}"
if [ "${{github.event_name}}" == "schedule" ]; then
if [[ "${{github.event_name}}" == "schedule" || "${{github.event_name}}" == "workflow_dispatch" ]]; then
echo 'meta_matrix=["sqlite3", "redis", "mysql", "tikv", "tidb", "postgres", "badger", "mariadb", "fdb"]' >> $GITHUB_OUTPUT
elif [ "${{github.event_name}}" == "pull_request" ]; then
echo 'meta_matrix=["redis"]' >> $GITHUB_OUTPUT
elif [ "${{github.event_name}}" == "workflow_dispatch" ]; then
elif [[ "${{github.event_name}}" == "pull_request" || "${{github.event_name}}" == "push" ]]; then
echo 'meta_matrix=["redis"]' >> $GITHUB_OUTPUT
else
echo 'meta_matrix=["redis"]' >> $GITHUB_OUTPUT
echo 'event name is not supported' && exit 1
fi
outputs:
meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }}
@@ -100,12 +98,8 @@ jobs:
meta_url=$(get_meta_url ${{matrix.meta}})
# sudo mkdir /var/jfs
# sudo chmod 777 /var/jfs
sudo ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report &
sleep 5
if [ ! -f /tmp/jfs/.accesslog ]; then
echo "<FATAL>: mount failed"
exit 1
fi
sudo ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report
stat /tmp/jfs/.accesslog
- name: Pjdfstest
run: |

145
.github/workflows/xattr.yml vendored Normal file
View File

@@ -0,0 +1,145 @@
name: "xattr"
on:
push:
branches:
- 'main'
- 'release-**'
paths:
- '**.go'
- '**.c'
- '**/xattr.yml'
pull_request:
branches:
- 'main'
- 'release-**'
paths:
- '**.go'
- '**.c'
- '**/xattr.yml'
schedule:
- cron: '0 19 * * *'
workflow_dispatch:
inputs:
debug:
type: boolean
description: "Run the build with tmate debugging enabled"
required: false
default: false
jobs:
build-matrix:
runs-on: ubuntu-20.04
steps:
- id: set-matrix
run: |
echo "github.event_name is ${{github.event_name}}"
if [[ "${{github.event_name}}" == "schedule" || "${{github.event_name}}" == "workflow_dispatch" ]]; then
echo 'meta_matrix=["sqlite3", "redis", "mysql", "postgres", "badger", "mariadb", "fdb"]' >> $GITHUB_OUTPUT
elif [[ "${{github.event_name}}" == "pull_request" || "${{github.event_name}}" == "push" ]]; then
echo 'meta_matrix=["redis"]' >> $GITHUB_OUTPUT
else
echo 'event name is not supported' && exit 1
fi
outputs:
meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }}
xattr:
needs: build-matrix
strategy:
fail-fast: false
matrix:
meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }}
runs-on: ubuntu-20.04
steps:
- uses: shogo82148/actions-setup-perl@v1
with:
perl-version: '5.34'
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 1
- name: Set Variable
id: vars
run: |
if [ "${{matrix.meta}}" == "fdb" ]; then
echo "target=juicefs.fdb" >> $GITHUB_OUTPUT
else
echo "target=juicefs" >> $GITHUB_OUTPUT
fi
- name: Build
uses: ./.github/actions/build
with:
target: ${{steps.vars.outputs.target}}
- name: Prepare meta db
run: |
chmod +x .github/scripts/start_meta_engine.sh
source .github/scripts/start_meta_engine.sh
start_meta_engine ${{matrix.meta}}
meta_url=$(get_meta_url ${{matrix.meta}})
create_database $meta_url
- name: Juicefs Format
run: |
source .github/scripts/start_meta_engine.sh
meta_url=$(get_meta_url ${{matrix.meta}})
sudo ./juicefs format $meta_url --trash-days 0 pics
- name: Juicefs Mount
run: |
source .github/scripts/start_meta_engine.sh
meta_url=$(get_meta_url ${{matrix.meta}})
# sudo mkdir /var/jfs
# sudo chmod 777 /var/jfs
sudo ./juicefs mount -d $meta_url /tmp/jfs --no-usage-report --enable-xattr
stat /tmp/jfs/.accesslog
- name: Test
run: |
git clone https://github.com/iustin/pyxattr.git
cd pyxattr
pip3 install pytest
pip3 install pyxattr
stat /tmp/jfs/
TEST_DIR=/tmp/jfs/ python3 -m pytest tests
- name: log
if: always()
run: |
if [ -f /var/log/juicefs.log ]; then
tail -300 /var/log/juicefs.log
grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
fi
- name: Setup upterm session
if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
timeout-minutes: 60
uses: lhotari/action-upterm@v1
success-all-test:
runs-on: ubuntu-latest
needs: [xattr]
if: always()
steps:
- uses: technote-space/workflow-conclusion-action@v3
- uses: actions/checkout@v3
- name: Check Failure
if: env.WORKFLOW_CONCLUSION == 'failure'
run: exit 1
- name: Send Slack Notification
if: failure() && github.event_name != 'workflow_dispatch'
uses: juicedata/slack-notify-action@main
with:
channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"
- name: Success
if: success()
run: echo "All Done"

3
.gitignore vendored
View File

@@ -5,9 +5,6 @@ ltmain.sh
*.rej
.deps
.dirstamp
fstests/secfs.test
fstests/flock
!fstests/Makefile
jfs
*.rdb
.release-env

2
.golangci.yml
View File

@@ -1,5 +1,3 @@
run:
timeout: 5m
tests: false
skip-dirs:
- fstests

43
fstests/Makefile
View File

@@ -1,43 +0,0 @@
DURATION ?= 10
all: fsracer fsx xattrs
xattrs:
touch /jfs/test_xattrs
setfattr -n user.k -v value /jfs/test_xattrs
getfattr -n user.k /jfs/test_xattrs | grep -q user.k=
fsracer: healthcheck secfs.test/tools/bin/fsracer
secfs.test/tools/bin/fsracer $(DURATION) /jfs >fsracer.log
make healthcheck
fsx: healthcheck secfs.test/tools/bin/fsx
secfs.test/tools/bin/fsx -d $(DURATION) -p 10000 -F 100000 /jfs/fsx.out
make healthcheck
setup:
redis-server &
cd node1 && redis-server redis.conf &
cd node2 && redis-server redis.conf &
cd node3 && redis-server redis.conf &
sleep 1
echo yes | redis-cli --cluster create 127.0.0.1:7001 127.0.0.1:7002 127.0.0.1:7003
mkdir -p /jfs
../juicefs format localhost unittest
../juicefs mount -d --no-usage-report --enable-xattr localhost /jfs
healthcheck:
pgrep juicefs
secfs.test/tools/bin/fsx: secfs.test
secfs.test/tools/bin/fsracer: secfs.test
secfs.test:
git clone https://github.com/billziss-gh/secfs.test.git
make -C secfs.test >secfs.test-build.log 2>&1
flock:
git clone https://github.com/gofrs/flock.git
mkdir /jfs/tmp
cd flock && go mod init github.com/gofrs/flock.git && go mod tidy && TMPDIR=/jfs/tmp go test .

5
fstests/node1/redis.conf
View File

@@ -1,5 +0,0 @@
port 7001
cluster-enabled yes
cluster-config-file nodes.conf
cluster-node-timeout 5000
appendonly yes

5
fstests/node2/redis.conf
View File

@@ -1,5 +0,0 @@
port 7002
cluster-enabled yes
cluster-config-file nodes.conf
cluster-node-timeout 5000
appendonly yes

5
fstests/node3/redis.conf
View File

@@ -1,5 +0,0 @@
port 7003
cluster-enabled yes
cluster-config-file nodes.conf
cluster-node-timeout 5000
appendonly yes