promptflow-sdk-cli-test #10077
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: promptflow-sdk-cli-test | |
| on: | |
| schedule: | |
| - cron: "40 18 * * *" # Every day starting at 2:40 BJT | |
| pull_request: | |
| paths: | |
| - src/promptflow-core/** | |
| - src/promptflow-devkit/** | |
| - src/promptflow/** | |
| - src/promptflow-tracing/** | |
| - scripts/building/** | |
| - .github/workflows/promptflow-sdk-cli-test.yml | |
| - src/promptflow-recording/** | |
| workflow_dispatch: | |
| env: | |
| IS_IN_CI_PIPELINE: "true" | |
| RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording | |
| TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing | |
| CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core | |
| WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-devkit | |
| PROMPTFLOW_DIRECTORY: ${{ github.workspace }}/src/promptflow | |
| TOOL_DIRECTORY: ${{ github.workspace }}/src/promptflow-tools | |
| jobs: | |
| sdk_cli_tests: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] | |
| pythonVersion: ['3.8', '3.9', '3.10', '3.11'] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: set test mode | |
| run: echo "PROMPT_FLOW_TEST_MODE=$(if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo replay; else echo live; fi)" >> $GITHUB_ENV | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.pythonVersion }} | |
| - uses: snok/install-poetry@v1 | |
| - name: install test dependency group | |
| run: | | |
| set -xe | |
| poetry install --only test | |
| poetry run pip install ${{ env.TRACING_DIRECTORY }} | |
| poetry run pip install ${{ env.CORE_DIRECTORY }}[azureml-serving] | |
| poetry run pip install -e ${{ env.WORKING_DIRECTORY }}[pyarrow] | |
| echo "Need to install promptflow to avoid tool dependency issue" | |
| poetry run pip install ${{ env.PROMPTFLOW_DIRECTORY }} | |
| poetry run pip install ${{ env.TOOL_DIRECTORY }} | |
| poetry run pip install -e ${{ env.RECORD_DIRECTORY }} | |
| poetry run pip show promptflow-tracing | |
| poetry run pip show promptflow-core | |
| poetry run pip show promptflow-devkit | |
| poetry run pip show promptflow-tools | |
| working-directory: ${{ env.WORKING_DIRECTORY }} | |
| - name: Azure login (non pull_request workflow) | |
| if: github.event_name != 'pull_request' | |
| uses: azure/login@v1 | |
| with: | |
| creds: ${{ secrets.AZURE_CREDENTIALS }} | |
| - name: generate live test resources (non pull_request workflow) | |
| if: github.event_name != 'pull_request' | |
| uses: "./.github/actions/step_generate_configs" | |
| with: | |
| targetFolder: ${{ env.PROMPTFLOW_DIRECTORY }} | |
| - name: generate live test resources (pull_request workflow) | |
| if: github.event_name == 'pull_request' | |
| working-directory: ${{ env.PROMPTFLOW_DIRECTORY }} | |
| run: | | |
| cp ${{ github.workspace }}/src/promptflow/dev-connections.json.example ${{ github.workspace }}/src/promptflow/connections.json | |
| - name: run devkit tests | |
| run: | | |
| poetry run pytest ./tests/sdk_cli_test ./tests/sdk_pfs_test -p promptflow --cov=promptflow --cov-config=pyproject.toml \ | |
| --cov-report=term --cov-report=html --cov-report=xml -n auto -m "unittest or e2etest" \ | |
| --ignore-glob ./tests/sdk_cli_test/e2etests/test_executable.py | |
| working-directory: ${{ env.WORKING_DIRECTORY }} | |
| - name: Upload Test Results | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Test Results (Python ${{ matrix.pythonVersion }}) (OS ${{ matrix.os }}) | |
| path: | | |
| ${{ env.WORKING_DIRECTORY }}/*.xml | |
| ${{ env.WORKING_DIRECTORY }}/htmlcov/ | |
| ${{ env.WORKING_DIRECTORY }}/tests/sdk_cli_test/count.json | |
| - run: poetry run pip install -e ${{ env.WORKING_DIRECTORY }}[executable] | |
| working-directory: ${{ env.WORKING_DIRECTORY }} | |
| - name: run devkit executable tests | |
| run: | | |
| poetry run pytest -n auto -m "unittest or e2etest" ./tests/sdk_cli_test/e2etests/test_executable.py | |
| working-directory: ${{ env.WORKING_DIRECTORY }} | |
| publish-test-results-sdk-cli-test: | |
| needs: sdk_cli_tests | |
| runs-on: ubuntu-latest | |
| permissions: | |
| checks: write | |
| pull-requests: write | |
| contents: read | |
| issues: read | |
| if: always() | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: Publish Test Results | |
| uses: "./.github/actions/step_publish_test_results" | |
| with: | |
| testActionFileName: promptflow-sdk-cli-test.yml | |
| testResultTitle: SDK CLI Test Result | |
| osVersion: ubuntu-latest | |
| pythonVersion: 3.9 | |
| coverageThreshold: 40 | |
| context: test/sdk_cli |