diff --git a/.github/workflows/_build_linux.yml b/.github/workflows/_build_linux.yml index a8f29fe7ed..5836862896 100644 --- a/.github/workflows/_build_linux.yml +++ b/.github/workflows/_build_linux.yml @@ -22,12 +22,22 @@ on: description: "Enable nightly build mode (e.g. add date suffix to version)" required: false type: string - default: "ON" + default: "OFF" FD_VERSION: description: "FastDeploy Package Version" required: false type: string default: "" + PADDLEVERSION: + description: "Paddle Version Build Use" + required: false + type: string + default: "" + PADDLE_WHL_URL: + description: "Paddle Wheel Package URL" + required: false + type: string + default: "" UPLOAD: description: "Upload Package" required: false @@ -85,6 +95,10 @@ jobs: compile_arch: ${{ inputs.COMPILE_ARCH }} fd_version: ${{ inputs.FD_VERSION }} CACHE_DIR: ${{ inputs.CACHE_DIR }} + BRANCH_REF: ${{ github.ref_name }} + PADDLEVERSION: ${{ inputs.PADDLEVERSION }} + PADDLE_WHL_URL: ${{ inputs.PADDLE_WHL_URL }} + WITH_NIGHTLY_BUILD: ${{ inputs.WITH_NIGHTLY_BUILD }} run: | set -x runner_name="${{ runner.name }}" @@ -109,6 +123,9 @@ jobs: -e "COMPILE_ARCH=${compile_arch}" \ -e "FD_VERSION=${fd_version}" \ -e "WITH_NIGHTLY_BUILD=${WITH_NIGHTLY_BUILD}" \ + -e "PADDLEVERSION=${PADDLEVERSION}" \ + -e "PADDLE_WHL_URL=${PADDLE_WHL_URL}" \ + -e "BRANCH_REF=${BRANCH_REF}" \ --gpus "\"device=${gpu_id}\"" ${docker_image} /bin/bash -c ' if [[ -n "${FD_VERSION}" ]]; then export FASTDEPLOY_VERSION=${FD_VERSION} @@ -124,7 +141,15 @@ jobs: echo "Date Only: $DATE_ONLY" export FASTDEPLOY_VERSION="${FASTDEPLOY_VERSION}.dev${DATE_ONLY}" fi - python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/ + # 针对不同分支和tag使用不同的PaddlePaddle安装包 + if [[ "${PADDLE_WHL_URL}" != "" ]];then + python -m pip install ${PADDLE_WHL_URL} + elif [[ "${PADDLEVERSION}" != "" ]];then + python -m pip install paddlepaddle-gpu==${PADDLEVERSION} -i https://www.paddlepaddle.org.cn/packages/stable/cu126/ + else + python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/ + fi + pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple python -m pip install --upgrade pip diff --git a/.github/workflows/_pre_ce_test.yml b/.github/workflows/_pre_ce_test.yml index 637eeb249f..b7415f9cb8 100644 --- a/.github/workflows/_pre_ce_test.yml +++ b/.github/workflows/_pre_ce_test.yml @@ -27,10 +27,6 @@ on: type: string default: "" -concurrency: - group: ${{ github.event.pull_request.number }} - cancel-in-progress: true - jobs: run_ce_cases: runs-on: [self-hosted, PRE_CE_RUN_2Card] diff --git a/.github/workflows/_unit_test_coverage.yml b/.github/workflows/_unit_test_coverage.yml index 99864c5017..7d02a1f9e5 100644 --- a/.github/workflows/_unit_test_coverage.yml +++ b/.github/workflows/_unit_test_coverage.yml @@ -73,8 +73,13 @@ jobs: CACHE_DIR: ${{ inputs.CACHE_DIR }} BASE_REF: ${{ github.event.pull_request.base.ref }} MODEL_CACHE_DIR: ${{ inputs.MODEL_CACHE_DIR }} + IS_PR: ${{ github.event_name == 'pull_request' }} run: | - set -x + if [[ "$IS_PR" == "true" ]]; then + echo "Running on PR" + else + echo "Not a PR" + fi runner_name="${{ runner.name }}" CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}') DEVICES=$(echo "$CARD_ID" | fold -w1 | paste -sd,) @@ -133,6 +138,7 @@ jobs: -e TZ="Asia/Shanghai" \ -e "fd_wheel_url=${fd_wheel_url}" \ -e "BASE_REF=${BASE_REF}" \ + -e "IS_PR=${IS_PR}" \ --gpus "\"device=${DEVICES}\"" ${docker_image} /bin/bash -c ' git config --global --add safe.directory /workspace/FastDeploy @@ -160,9 +166,13 @@ jobs: coverage combine coveragedata/ coverage xml -o python_coverage_all.xml COVERAGE_EXIT_CODE=0 - diff-cover python_coverage_all.xml --diff-file=diff.txt --fail-under=80 --json-report diff_coverage.json || COVERAGE_EXIT_CODE=9 + if [[ "$IS_PR" == "true" ]]; then + diff-cover python_coverage_all.xml --diff-file=diff.txt --fail-under=80 --json-report diff_coverage.json || COVERAGE_EXIT_CODE=9 + python scripts/generate_diff_coverage_xml.py diff.txt python_coverage_all.xml + else + echo "Not a PR, skipping diff-cover" + fi echo "COVERAGE_EXIT_CODE=${COVERAGE_EXIT_CODE}" >> exit_code.env - python scripts/generate_diff_coverage_xml.py diff.txt python_coverage_all.xml ' if [ -f FastDeploy/exit_code.env ]; then cat FastDeploy/exit_code.env >> $GITHUB_ENV @@ -224,6 +234,7 @@ jobs: echo "All tests passed" - name: Verify Code Coverage Threshold (80%) + if: ${{ github.event_name == 'pull_request' }} shell: bash run: | cd FastDeploy diff --git a/.github/workflows/publish_job.yml b/.github/workflows/publish_job.yml new file mode 100644 index 0000000000..76082667b6 --- /dev/null +++ b/.github/workflows/publish_job.yml @@ -0,0 +1,319 @@ +name: Publish Job + +on: + workflow_dispatch: + schedule: + - cron: '0 18 * * *' # 2:00 AM China Standard Time (UTC+8) + push: + # branches: + # - develop + tags: + - '*' + +permissions: read-all + +concurrency: + group: ${{ github.ref }}-${{ github.sha }} + cancel-in-progress: true + + +jobs: + publish_pre_check: + runs-on: ubuntu-latest + if: | + github.event.repository.fork == false && + ( + (github.event_name == 'schedule' && github.ref_name == 'develop') || + (github.event_name == 'push' && github.ref_type == 'tag') || + ((github.event_name == 'workflow_dispatch') && + (github.ref_name == 'develop' || github.ref_type == 'tag')) + ) + env: + TAG_VERSION_MAPPINGS: ${{ vars.TAG_VERSION_MAPPINGS }} + FD_VERSION_DEV: ${{ vars.FD_VERSION_DEV }} + COMPILE_USE_PADDLE_WHL_URL_MAPPINGS: ${{ vars.COMPILE_USE_PADDLE_WHL_URL_MAPPINGS }} + outputs: + compile_use_paddle_version: ${{ steps.set_output.outputs.compile_use_paddle_version }} + compile_continue: ${{ steps.set_output.outputs.compile_continue }} + fd_version: ${{ steps.set_output.outputs.fd_version }} + with_nightly_build: ${{ steps.set_output.outputs.with_nightly_build }} + compile_use_paddle_whl_url: ${{ steps.set_output.outputs.compile_use_paddle_whl_url }} + + steps: + - name: Get tag version + if: github.ref_type == 'tag' + run: | + TAG_NAME="${GITHUB_REF##*/}" # 提取 tag 名称,比如 v2.1.0 + TAG_VERSION="${TAG_NAME#v}" # 去掉前缀 v + echo "FD_VERSION=$TAG_VERSION" >> $GITHUB_ENV + + - name: Check FD version to Paddle version mapping + if: github.ref_type == 'tag' + env: + TARGET_FD: ${{ env.FD_VERSION }} + run: | + FOUND_PADDLE="" + # 遍历映射 + for pair in $(echo $TAG_VERSION_MAPPINGS | tr ';' ' '); do + fd=$(echo "$pair" | cut -d',' -f1) + paddle=$(echo "$pair" | cut -d',' -f2) + if [[ "$fd" == "$TARGET_FD" ]]; then + FOUND_PADDLE="$paddle" + break + fi + done + + if [[ -z "$FOUND_PADDLE" ]]; then + echo "No Paddle version found for FD $TARGET_FD" + else + echo "FD $TARGET_FD maps to Paddle $FOUND_PADDLE" + echo "PADDLE_VERSION=$FOUND_PADDLE" >> $GITHUB_ENV + fi + - name: Set Version + id: set_output + env: + PADDLE_VERSION: ${{ env.PADDLE_VERSION }} + FD_VERSION: ${{ env.FD_VERSION }} + run: | + if [[ "${{ github.ref_type }}" == "tag" ]]; then + if [[ -z "$PADDLE_VERSION" ]]; then + compile_continue=false + else + compile_use_paddle_version=$PADDLE_VERSION + compile_continue=true + fi + fd_version=$FD_VERSION + fi + if [[ "${{ github.ref_name }}" == "develop" ]];then + compile_continue=true + compile_use_paddle_version="" + fd_version=${FD_VERSION_DEV} + with_nightly_build=ON + fi + # Todo + # 通过变量COMPILE_USE_PADDLE_WHL_URL_MAPPINGS中的映射关系,决定是否是安装指定版本的Paddle还是直接安装URL + for pair in $(echo $COMPILE_USE_PADDLE_WHL_URL_MAPPINGS | tr ';' ' '); do + branch=$(echo "$pair" | cut -d',' -f1) + paddle_whl_url=$(echo "$pair" | cut -d',' -f2) + if [[ "$branch" == "${{ github.ref_name }}" ]]; then + FOUND_PADDLE_URL="$paddle_whl_url" + echo "compile_use_paddle_whl_url=${FOUND_PADDLE_URL}" >> $GITHUB_OUTPUT + compile_continue=true + break + fi + done + echo "compile_continue=${compile_continue}" >> $GITHUB_OUTPUT + echo "compile_use_paddle_version=${compile_use_paddle_version}" >> $GITHUB_OUTPUT + echo "fd_version=${fd_version}" >> $GITHUB_OUTPUT + echo "with_nightly_build=${with_nightly_build:-OFF}" >> $GITHUB_OUTPUT + + print_publish_pre_check_outputs: + runs-on: ubuntu-latest + needs: publish_pre_check + steps: + - name: Print outputs as JSON + run: | + echo '${{ toJSON(needs.publish_pre_check.outputs) }}' + + clone: + environment: CodeSync + name: FD-Clone-Linux + runs-on: ubuntu-latest + needs: publish_pre_check + if: ${{ needs.publish_pre_check.outputs.compile_continue == 'true' }} + outputs: + repo_archive_url: ${{ steps.set_output.outputs.repo_archive_url }} + steps: + - name: Clone FastDeploy + uses: actions/checkout@v4 + with: + ref: ${{ github.ref_name }} + submodules: 'recursive' + fetch-depth: 1000 + + - name: Python Setup + uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Code Info Show and Upload + id: set_output + env: + AK: ${{ secrets.BOS_AK }} + SK: ${{ secrets.BOS_SK }} + run: | + git config --unset http.https://github.com/.extraheader + git submodule foreach --recursive sh -c "git config --local --unset-all 'http.https://github.com/.extraheader'" + git submodule foreach --recursive sh -c "git config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'" + echo "Current HEAD Log:" + git log --oneline -n 5 + ls + cd .. + tar -zcf FastDeploy.tar.gz FastDeploy + if [[ "${{ github.ref_type }}" == "tag" ]]; then + commit_id=${{ github.sha }} + tag_name=${{ github.ref_name }} + target_path=paddle-qa/TAG/FastDeploy/${tag_name}/${commit_id} + else + commit_id=${{ github.sha }} + branch_name=${{ github.ref_name }} + target_path=paddle-qa/BRANCH/FastDeploy/${branch_name}/${commit_id} + fi + wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py + push_file=$(realpath bos_tools.py) + python -m pip install bce-python-sdk==0.9.29 + ls + python ${push_file} FastDeploy.tar.gz ${target_path} + target_path_stripped="${target_path#paddle-qa/}" + REPO_ARCHIVE_URL=https://paddle-qa.bj.bcebos.com/${target_path_stripped}/FastDeploy.tar.gz + echo "repo_archive_url=${REPO_ARCHIVE_URL}" >> $GITHUB_OUTPUT + + resultshow: + name: Show Code Archive Output + needs: clone + runs-on: ubuntu-latest + steps: + - name: Print wheel path + run: | + echo "The code archive is located at: ${{ needs.clone.outputs.repo_archive_url }}" + + build_sm8090: + name: BUILD_SM8090 + needs: [clone, publish_pre_check] + uses: ./.github/workflows/_build_linux.yml + with: + DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate + FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} + COMPILE_ARCH: "80,90" + WITH_NIGHTLY_BUILD: ${{ needs.publish_pre_check.outputs.with_nightly_build }} + FD_VERSION: ${{ needs.publish_pre_check.outputs.fd_version }} + PADDLEVERSION: ${{ needs.publish_pre_check.outputs.compile_use_paddle_version }} + PADDLE_WHL_URL: ${{ needs.publish_pre_check.outputs.compile_use_paddle_whl_url }} + + build_sm8689: + name: BUILD_SM8689 + needs: [clone, publish_pre_check] + uses: ./.github/workflows/_build_linux.yml + with: + DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate + FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} + COMPILE_ARCH: "86,89" + WITH_NIGHTLY_BUILD: ${{ needs.publish_pre_check.outputs.with_nightly_build }} + FD_VERSION: ${{ needs.publish_pre_check.outputs.fd_version }} + PADDLEVERSION: ${{ needs.publish_pre_check.outputs.compile_use_paddle_version }} + PADDLE_WHL_URL: ${{ needs.publish_pre_check.outputs.compile_use_paddle_whl_url }} + + paddle_pypi_upload_sm8090: + environment: PaddleSourceUpload + name: PADDLE_PYPI_UPLOAD_8090 + needs: build_sm8090 + runs-on: ubuntu-latest + env: + AK: ${{ secrets.BOS_AK }} + SK: ${{ secrets.BOS_SK }} + FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }} + COMPILE_ARCH: "80,90" + steps: + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Wheel Info Show and Upload + if: github.ref_name == 'develop' || github.ref_type == 'tag' + run: | + echo "The wheel is located at: ${FASTDEPLOY_WHEEL_URL}" + wget -q --no-check-certificate ${FASTDEPLOY_WHEEL_URL} + filename=$(basename ${FASTDEPLOY_WHEEL_URL}) + if [[ "${{ github.ref_name }}" == "develop" ]];then + target_path=paddle-whl/nightly/fastdeploy-gpu-${COMPILE_ARCH//,/_}/fastdeploy-gpu + elif [[ "${{ github.ref_type }}" == "tag" ]]; then + target_path=paddle-whl/stable/fastdeploy-gpu-${COMPILE_ARCH//,/_}/fastdeploy-gpu + else + echo "Not develop or tag, do nothing" + fi + wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py + push_file=$(realpath bos_tools.py) + python -m pip install bce-python-sdk==0.9.29 + ls + python ${push_file} ${filename} ${target_path} + + paddle_pypi_upload_sm8689: + environment: PaddleSourceUpload + name: PADDLE_PYPI_UPLOAD_8689 + needs: build_sm8689 + runs-on: ubuntu-latest + env: + AK: ${{ secrets.BOS_AK }} + SK: ${{ secrets.BOS_SK }} + FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8689.outputs.wheel_path }} + COMPILE_ARCH: "86,89" + steps: + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Wheel Info Show and Upload + if: github.ref_name == 'develop' || github.ref_type == 'tag' + run: | + echo "The wheel is located at: ${FASTDEPLOY_WHEEL_URL}" + wget -q --no-check-certificate ${FASTDEPLOY_WHEEL_URL} + filename=$(basename ${FASTDEPLOY_WHEEL_URL}) + if [[ "${{ github.ref_name }}" == "develop" ]];then + target_path=paddle-whl/nightly/fastdeploy-gpu-${COMPILE_ARCH//,/_}/fastdeploy-gpu + elif [[ "${{ github.ref_type }}" == "tag" ]]; then + target_path=paddle-whl/stable/fastdeploy-gpu-${COMPILE_ARCH//,/_}/fastdeploy-gpu + else + echo "Not develop or tag, do nothing" + fi + wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py + push_file=$(realpath bos_tools.py) + python -m pip install bce-python-sdk==0.9.29 + ls + python ${push_file} ${filename} ${target_path} + + unittest_coverage: + name: Run FastDeploy Unit Tests and Coverage + needs: [clone,build_sm8090] + uses: ./.github/workflows/_unit_test_coverage.yml + with: + DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate + FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} + FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }} + MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" + + logprob_test: + name: Run FastDeploy LogProb Tests + needs: [build_sm8090] + uses: ./.github/workflows/_logprob_test_linux.yml + with: + DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate + PADDLETEST_ARCHIVE_URL: "https://xly-devops.bj.bcebos.com/PaddleTest/PaddleTest.tar.gz" + FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }} + MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" + + pre_ce_test: + name: Extracted partial CE model tasks to run in CI. + needs: [clone,build_sm8090] + uses: ./.github/workflows/_pre_ce_test.yml + with: + DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate + FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} + FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }} + MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" + + base_test: + name: Run Base Tests + needs: [clone,build_sm8090] + uses: ./.github/workflows/_base_test.yml + with: + DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate + FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} + FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }} + MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" + + accuracy_test: + name: Run Accuracy Tests + needs: [clone,build_sm8090] + uses: ./.github/workflows/_accuracy_test.yml + with: + DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:fastdeploy-ciuse-cuda126-dailyupdate + FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} + FASTDEPLOY_WHEEL_URL: ${{ needs.build_sm8090.outputs.wheel_path }} + MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelData" diff --git a/tools/dockerfile/Dockerfile.ci b/tools/dockerfile/Dockerfile.ci index 1afb1b987a..197efb3ffc 100644 --- a/tools/dockerfile/Dockerfile.ci +++ b/tools/dockerfile/Dockerfile.ci @@ -1,5 +1,3 @@ -FROM ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda126-dev -RUN apt update && apt install -y lsof -RUN wget https://raw.githubusercontent.com/PaddlePaddle/FastDeploy/refs/heads/develop/requirements.txt +FROM ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:cuda126-py310-cibase +COPY ../../requirements.txt ./requirements.txt RUN python -m pip install -r requirements.txt -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple && python -m pip install pytest -RUN apt update && apt install -y python3.10-venv