From c2e08107561e58a25165d34c649b2e6a3a531002 Mon Sep 17 00:00:00 2001 From: Nabil Hachicha Date: Wed, 10 Jan 2024 14:37:33 +0000 Subject: [PATCH] [CI] Sync tests with Device farm (#1605) --- .../run-android-device-farm-test/action.yml | 27 +- .github/workflows/include-check-cache.yml | 34 +++ .github/workflows/pr.yml | 285 +++++++++++++++++- CHANGELOG.md | 1 + .../src/androidMain/AndroidManifest.xml | 4 +- 5 files changed, 341 insertions(+), 10 deletions(-) diff --git a/.github/actions/run-android-device-farm-test/action.yml b/.github/actions/run-android-device-farm-test/action.yml index 084001cb46..09b371254b 100644 --- a/.github/actions/run-android-device-farm-test/action.yml +++ b/.github/actions/run-android-device-farm-test/action.yml @@ -2,7 +2,12 @@ name: 'Run Android tests on Device Farm' inputs: apk-path: required: true + apk-auxiliary-path: + description: 'Install additional APKs needed for the tests' + default: '' + required: false app-id: + description: 'The test runner class to use' required: true project-arn: required: true @@ -17,12 +22,14 @@ runs: using: "composite" steps: - name: Run the tests - uses: realm/aws-devicefarm/test-application@7b9a91236c456c97e28d384c9e476035d5ea686b + uses: nhachicha/aws-devicefarm/test-application@master + id: run-tests with: project_arn: ${{ inputs.project-arn }} device_pool_arn: ${{ inputs.device-pool-arn }} app_file: ${{ inputs.apk-path }} + app_auxiliary_files: ${{ inputs.apk-auxiliary-path }} app_type: ANDROID_APP test_type: APPIUM_PYTHON test_package_file: https://github.com/realm/aws-devicefarm-sample-data/releases/download/0.3/sample_env_python3.zip @@ -38,7 +45,8 @@ runs: - export PYTHON_VERSION=3 test: commands: - - adb shell am instrument -w -r io.realm.testapp.test/androidx.test.runner.AndroidJUnitRunner | egrep 'OK \([0-9]+ test[s]?\)' + - adb shell pm list packages | grep realm + - adb shell am instrument -w -r ${{ inputs.app-id }}/androidx.test.runner.AndroidJUnitRunner | egrep 'OK \([0-9]+ test[s]?\)' - run: | Install-Module -Name AWSPowerShell -Force @@ -53,11 +61,16 @@ runs: echo "All File Artifacts:" echo $fileArtifacts $logCatArtifacts = $fileArtifacts | Where-Object { $_.Name -EQ "Logcat" } - echo "LogCat Artifacts:" - echo $logCatArtifacts - echo "::group::Logcat" - Invoke-WebRequest -Uri $logCatArtifacts[0].Url | Select-Object -Expand RawContent - echo "::endgroup::" + if ($logCatArtifacts) { + echo "LogCat Artifacts:" + echo $logCatArtifacts + echo "::group::Logcat" + Invoke-WebRequest -Uri $logCatArtifacts[0].Url | Select-Object -Expand RawContent + echo "::endgroup::" + } else { + Write-Warning "No logCatArtifacts found." + } + shell: pwsh if: always() name: Device Farm Output diff --git a/.github/workflows/include-check-cache.yml b/.github/workflows/include-check-cache.yml index dab536b5b2..f5e8d97bfd 100644 --- a/.github/workflows/include-check-cache.yml +++ b/.github/workflows/include-check-cache.yml @@ -28,6 +28,8 @@ on: value: ${{ jobs.check-cache.outputs.packages-android-cache-hit }} android-test-base-apk-cache-hit: value: ${{ jobs.check-cache.outputs.android-test-base-apk-cache-hit }} + android-test-sync-apk-cache-hit: + value: ${{ jobs.check-cache.outputs.android-test-sync-apk-cache-hit }} packages-macos-x64-cache-hit: value: ${{ jobs.check-cache.outputs.packages-macos-x64-cache-hit }} packages-macos-arm64-cache-hit: @@ -63,6 +65,7 @@ jobs: packages-jvm-cache-hit: ${{ steps.jvm-cache.outputs.cache-hit }} packages-android-cache-hit: ${{ steps.android-cache.outputs.cache-hit }} android-test-base-apk-cache-hit: ${{ steps.android-test-base-apk.outputs.cache-hit }} + android-test-sync-apk-cache-hit: ${{ steps.android-test-sync-apk.outputs.cache-hit }} packages-macos-x64-cache-hit: ${{ steps.macos-x64-cache.outputs.cache-hit }} packages-macos-arm64-cache-hit: ${{ steps.macos-arm64-cache.outputs.cache-hit }} packages-ios-x64-cache-hit: ${{ steps.ios-x64-cache.outputs.cache-hit }} @@ -276,6 +279,18 @@ jobs: path: ./packages/test-base/build/outputs/apk/androidTest/debug/test-base-debug-androidTest.apk key: android-base-test-apk-key-${{ steps.packages-cache-key.outputs.sha }} + # + # Android Sync Test APK + # + - name: Check Android Sync Test APK + id: android-test-sync-apk + uses: cmelchior/cache@main + with: + key: android-sync-test-apk-key-${{ steps.packages-cache-key.outputs.sha }} + path: | + ./packages/test-sync/build/outputs/apk/androidTest/debug/test-sync-debug-androidTest.apk + ./packages/test-sync/build/outputs/apk/debug/test-sync-debug.apk + - name: Save Android Test APK uses: actions/upload-artifact@v3 if: always() && !cancelled() && steps.android-test-base-apk.outputs.cache-hit == 'true' @@ -284,6 +299,16 @@ jobs: path: ./packages/test-base/build/outputs/apk/androidTest/debug/test-base-debug-androidTest.apk retention-days: 1 + - name: Save Android Sync Test APK + uses: actions/upload-artifact@v3 + if: always() && !cancelled() && steps.android-test-sync-apk.outputs.cache-hit == 'true' + with: + name: android-sync-test-apk-${{ steps.find-library-version.outputs.label }} + retention-days: 1 + path: | + ./packages/test-sync/build/outputs/apk/androidTest/debug/test-sync-debug-androidTest.apk + ./packages/test-sync/build/outputs/apk/debug/test-sync-debug.apk + - name: Delete Android Test APK cache files id: delete-cache-android-base-test-apk uses: JesseTG/rm@v1.0.3 @@ -291,6 +316,15 @@ jobs: with: path: ./packages/test-base/build/outputs/apk/androidTest/debug/test-base-debug-androidTest.apk + - name: Delete Android Sync Test APK cache files + id: delete-cache-android-sync-test-apk + uses: JesseTG/rm@v1.0.3 + if: always() && !cancelled() && steps.android-test-sync-apk.outputs.cache-hit == 'true' + with: + path: | + ./packages/test-sync/build/outputs/apk/androidTest/debug/test-sync-debug-androidTest.apk + ./packages/test-sync/build/outputs/apk/debug/test-sync-debug.apk + # # MacOS arm64 # diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index bf743ff4c7..6b17ec9638 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1,5 +1,6 @@ name: PR Build on: + workflow_dispatch: # Add this line to enable manual triggering pull_request: paths-ignore: - '**.md' @@ -50,7 +51,7 @@ jobs: uses: jwlawson/actions-setup-cmake@v1.13 with: cmake-version: '3.27.7' - + # Manually install SWIG 4.1.0 as only 4.0.2 is pre-installed # 4.1.0 is not available in apt-get, so use brew instead # https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2204-Readme.md#ubuntu-22041-lts @@ -552,6 +553,27 @@ jobs: echo "/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH echo "/home/linuxbrew/.linuxbrew/bin" >> $GITHUB_PATH + - name: Install JSON parser + run: brew install jq + + # checkout BAAS CLI repo + - name: Checkout BAAS repo + run: | + echo ${{ secrets.BAAS_CLI }} | gh auth login --with-token + gh repo clone 10gen/baasaas + + # Start BAAS instance in the background + # We save the container id to poll against and get the hostname info later + - name: Start Baas instance in the background + id: baas_cli_start + working-directory: baasaas + env: + APIKEY: ${{ secrets.BAAS_CLI_API_KEY }} + run: | + OUTPUT=$(bash cli.sh start | jq -r '.id') + echo "baas_container_id=$OUTPUT" >> $GITHUB_OUTPUT + + # TODO This cmake version is not being used by the Android builds. Figure out why. - name: Setup cmake uses: jwlawson/actions-setup-cmake@v1.13 with: @@ -599,6 +621,20 @@ jobs: working-directory: packages run: ./gradlew :test-base:assembleAndroidTest -Prealm.kotlin.buildRealmCore=false -Prealm.kotlin.mainHost=false + # We poll the previously started BAAS container to get the hostname of the container to use with Device Farm tests + - name: Fetching the BAAS CLI hostname + id: baas_cli_poll + working-directory: baasaas + env: + APIKEY: ${{ secrets.BAAS_CLI_API_KEY }} + run: | + OUTPUT=$(bash cli.sh poll ${{ steps.baas_cli_start.outputs.baas_container_id }} | jq -r '.httpUrl') + echo "baas_container_hostname=$OUTPUT" >> $GITHUB_OUTPUT + + - name: Build Android Sync Test Apk + working-directory: packages + run: ./gradlew -PsyncTestUrl=${{ steps.baas_cli_poll.outputs.baas_container_hostname }} :test-sync:packageDebug :test-sync:assembleAndroidTest -Prealm.kotlin.buildRealmCore=false -Prealm.kotlin.mainHost=false + - name: Build packages working-directory: packages run: ./gradlew publishCIPackages -Prealm.kotlin.targets=android -Prealm.kotlin.buildRealmCore=false -Prealm.kotlin.mainHost=false @@ -620,6 +656,14 @@ jobs: path: ./packages/test-base/build/outputs/apk/androidTest/debug/test-base-debug-androidTest.apk key: android-base-test-apk-key-${{ needs.check-cache.outputs.packages-sha }} + - name: Store build cache for Android Sync Test APK + uses: actions/cache@v3 + with: + key: android-sync-test-apk-key-${{ needs.check-cache.outputs.packages-sha }} + path: | + ./packages/test-sync/build/outputs/apk/androidTest/debug/test-sync-debug-androidTest.apk + ./packages/test-sync/build/outputs/apk/debug/test-sync-debug.apk + # TODO Must match naming found in include-check-cache.yml - name: Upload artifacts uses: actions/upload-artifact@v3 @@ -635,6 +679,18 @@ jobs: path: ./packages/test-base/build/outputs/apk/androidTest/debug/test-base-debug-androidTest.apk retention-days: 1 + - name: Upload Android Sync Test APK + uses: actions/upload-artifact@v3 + with: + name: android-sync-test-apk-${{ needs.check-cache.outputs.version-label }} + retention-days: 1 + path: | + ./packages/test-sync/build/outputs/apk/androidTest/debug/test-sync-debug-androidTest.apk + ./packages/test-sync/build/outputs/apk/debug/test-sync-debug.apk + + outputs: + baas-container-id: ${{ steps.baas_cli_start.outputs.baas_container_id }} + # TODO: ccache is not being used by this build for some reason build-macos-x64-packages: runs-on: macos-latest @@ -1104,6 +1160,60 @@ jobs: project-arn: ${{ secrets.DEVICEFARM_PROJECT_ARN }} device-pool-arn: ${{ secrets.DEVICEFARM_POOL_ARN }} + test-android-packages-device-farm-sync: + name: AWS Device Farm Sync Tests + timeout-minutes: 60 + runs-on: ubuntu-latest + needs: [ check-cache, build-android-packages, build-jvm-packages ] + if: | + always() && + !cancelled() && + !contains(needs.*.result, 'failure') && + !contains(needs.*.result, 'cancelled') + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Restore Android Sync Test APK + uses: actions/download-artifact@v3 + with: + name: android-sync-test-apk-${{ needs.check-cache.outputs.version-label }} + path: ./packages/test-sync/build/outputs/apk/ + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_DEVICEFARM_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_DEVICEFARM_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + + - name: Run the Sync tests + uses: ./.github/actions/run-android-device-farm-test + id: run_android_tests + with: + apk-path: ${{ github.workspace }}/packages/test-sync/build/outputs/apk/androidTest/debug/test-sync-debug-androidTest.apk + apk-auxiliary-path: ${{ github.workspace }}/packages/test-sync/build/outputs/apk/debug/test-sync-debug.apk + app-id: io.realm.sync.testapp.test + project-arn: ${{ secrets.DEVICEFARM_PROJECT_ARN }} + device-pool-arn: ${{ secrets.DEVICEFARM_POOL_ARN }} + + - name: Checkout BAAS repo + if: always() + run: | + echo ${{ secrets.BAAS_CLI }} | gh auth login --with-token + gh repo clone 10gen/baasaas + + - name: Stopping the BAAS container + if: always() + working-directory: baasaas + env: + APIKEY: ${{ secrets.BAAS_CLI_API_KEY }} + run: | + if [ -n "${{ needs.build-android-packages.outputs.baas-container-id }}" ]; then + bash cli.sh stop ${{ needs.build-android-packages.outputs.baas-container-id }} + fi + test-macos-packages: timeout-minutes: 30 strategy: @@ -1132,6 +1242,25 @@ jobs: - name: Checkout code uses: actions/checkout@v3 + # checkout BAAS CLI repo + - name: Checkout BAAS repo + run: | + echo ${{ secrets.BAAS_CLI }} | gh auth login --with-token + gh repo clone 10gen/baasaas + + # Start BAAS instance in the background + # We save the container id to poll against and get the hostname info later + - name: Start Baas instance in the background + id: baas_cli_start + working-directory: baasaas + env: + APIKEY: ${{ secrets.BAAS_CLI_API_KEY }} + run: | + # Adding a dummy tag (foo=bar) to avoid the following issue on macos-runner + # curl: option --data: error encountered when reading a file + OUTPUT=$(bash cli.sh start -t foo=bar | jq -r '.id') + echo "baas_container_id=$OUTPUT" >> $GITHUB_OUTPUT + - name: Setup Java 11 uses: actions/setup-java@v3 with: @@ -1167,8 +1296,44 @@ jobs: reporter: java-junit list-suites: failed list-tests: failed + fail-on-error: true + + # We poll the previously started BAAS container to get the hostname of the container to use with Device Farm tests + - name: Fetching the BAAS CLI hostname + id: baas_cli_poll + working-directory: baasaas + env: + APIKEY: ${{ secrets.BAAS_CLI_API_KEY }} + run: | + OUTPUT=$(bash cli.sh poll ${{ steps.baas_cli_start.outputs.baas_container_id }} | jq -r '.httpUrl') + echo "baas_container_hostname=$OUTPUT" >> $GITHUB_OUTPUT + + - name: Run Sync tests + working-directory: packages + run: | + ./gradlew -PsyncTestUrl=${{ steps.baas_cli_poll.outputs.baas_container_hostname }} :test-sync:macosTest -PincludeSdkModules=false --info --no-daemon + + - name: Publish Sync Unit Test Results + uses: dorny/test-reporter@v1 + if: always() || failure() + with: + name: "Sync ${{ matrix.test-title }}" + path: ./packages/test-sync/build/**/TEST-*.xml + reporter: java-junit + list-suites: failed + list-tests: failed fail-on-error: true + - name: Stopping the BAAS container + if: always() + working-directory: baasaas + env: + APIKEY: ${{ secrets.BAAS_CLI_API_KEY }} + run: | + if [ -n "${{ steps.baas_cli_start.outputs.baas_container_id }}" ]; then + bash cli.sh stop ${{ steps.baas_cli_start.outputs.baas_container_id }} + fi + test-ios-packages: timeout-minutes: 30 strategy: @@ -1197,6 +1362,25 @@ jobs: - name: Checkout code uses: actions/checkout@v3 + # checkout BAAS CLI repo + - name: Checkout BAAS repo + run: | + echo ${{ secrets.BAAS_CLI }} | gh auth login --with-token + gh repo clone 10gen/baasaas + + # Start BAAS instance in the background + # We save the container id to poll against and get the hostname info later + - name: Start Baas instance in the background + id: baas_cli_start + working-directory: baasaas + env: + APIKEY: ${{ secrets.BAAS_CLI_API_KEY }} + run: | + # Adding a dummy tag (foo=bar) to avoid the following issue on macos-runner + # curl: option --data: error encountered when reading a file + OUTPUT=$(bash cli.sh start | jq -r '.id') + echo "baas_container_id=$OUTPUT" >> $GITHUB_OUTPUT + - name: Setup Java 11 uses: actions/setup-java@v3 with: @@ -1240,6 +1424,42 @@ jobs: list-tests: failed fail-on-error: true + # We poll the previously started BAAS container to get the hostname of the container to use with Device Farm tests + - name: Fetching the BAAS CLI hostname + id: baas_cli_poll + working-directory: baasaas + env: + APIKEY: ${{ secrets.BAAS_CLI_API_KEY }} + run: | + OUTPUT=$(bash cli.sh poll ${{ steps.baas_cli_start.outputs.baas_container_id }} | jq -r '.httpUrl') + echo "baas_container_hostname=$OUTPUT" >> $GITHUB_OUTPUT + + - name: Run Sync tests + working-directory: packages + run: | + ./gradlew -PsyncTestUrl=${{ steps.baas_cli_poll.outputs.baas_container_hostname }} :test-sync:iosTest -PincludeSdkModules=false --info --no-daemon + + - name: Publish Sync Unit Test Results + uses: dorny/test-reporter@v1 + if: always() || failure() + with: + name: "Sync ${{ matrix.test-title }}" + path: ./packages/test-sync/build/**/TEST-*.xml + reporter: java-junit + list-suites: failed + list-tests: failed + fail-on-error: true + + - name: Stopping the BAAS container + if: always() + working-directory: baasaas + env: + APIKEY: ${{ secrets.BAAS_CLI_API_KEY }} + run: | + if [ -n "${{ steps.baas_cli_start.outputs.baas_container_id }}" ]; then + bash cli.sh stop ${{ steps.baas_cli_start.outputs.baas_container_id }} + fi + # TODO Investigate why these tests seem to hang for the `run tests` step with no log output. Could it be related to the test failing, but not being picked up? test-jvm-packages: timeout-minutes: 30 @@ -1266,6 +1486,27 @@ jobs: - name: Checkout code uses: actions/checkout@v3 + # checkout BAAS CLI repo + - name: Checkout BAAS repo + if: matrix.os != 'windows-latest' + run: | + echo ${{ secrets.BAAS_CLI }} | gh auth login --with-token + gh repo clone 10gen/baasaas + + # Start BAAS instance in the background + # We save the container id to poll against and get the hostname info later + - name: Start Baas instance in the background + if: matrix.os != 'windows-latest' + id: baas_cli_start + working-directory: baasaas + env: + APIKEY: ${{ secrets.BAAS_CLI_API_KEY }} + run: | + # Adding a dummy tag (foo=bar) to avoid the following issue on macos-runner + # curl: option --data: error encountered when reading a file + OUTPUT=$(bash cli.sh start | jq -r '.id') + echo "baas_container_id=$OUTPUT" >> $GITHUB_OUTPUT + - name: Setup Java 11 uses: actions/setup-java@v3 with: @@ -1304,7 +1545,46 @@ jobs: list-tests: failed fail-on-error: true - package-all-artifacts: + # We poll the previously started BAAS container to get the hostname of the container to use with Device Farm tests + - name: Fetching the BAAS CLI hostname + if: matrix.os != 'windows-latest' + id: baas_cli_poll + working-directory: baasaas + env: + APIKEY: ${{ secrets.BAAS_CLI_API_KEY }} + run: | + OUTPUT=$(bash cli.sh poll ${{ steps.baas_cli_start.outputs.baas_container_id }} | jq -r '.httpUrl') + echo "baas_container_hostname=$OUTPUT" >> $GITHUB_OUTPUT + + - name: Run Sync tests + if: matrix.os != 'windows-latest' + working-directory: packages + run: | + ./gradlew -PsyncTestUrl=${{ steps.baas_cli_poll.outputs.baas_container_hostname }} :test-sync:jvmTest -PincludeSdkModules=false --info --no-daemon + + - name: Publish Sync Unit Test Results + uses: dorny/test-reporter@v1 + if: ${{ !cancelled() && matrix.os != 'windows-latest' }} + with: + name: "Sync - ${{ matrix.test-title }}" + path: ./packages/test-sync/build/**/TEST-*.xml + reporter: java-junit + list-suites: failed + list-tests: failed + fail-on-error: true + + - name: Stopping the BAAS container + if: always() && matrix.os != 'windows-latest' + working-directory: baasaas + env: + APIKEY: ${{ secrets.BAAS_CLI_API_KEY }} + run: | + if [ -n "${{ steps.baas_cli_start.outputs.baas_container_id }}" ]; then + bash cli.sh stop ${{ steps.baas_cli_start.outputs.baas_container_id }} + fi + + + package-all-artifacts: runs-on: ubuntu-latest needs: [check-cache, build-jvm-packages, build-android-packages, build-macos-x64-packages, build-macos-arm64-packages, build-ios-x64-packages, build-ios-arm64-packages, build-kotlin-metadata-package] if: | @@ -1390,6 +1670,7 @@ jobs: test-ios-packages, test-android-packages-emulator, test-android-packages-device-farm, + test-android-packages-device-farm-sync, package-all-artifacts ] if: | diff --git a/CHANGELOG.md b/CHANGELOG.md index ebee13f301..283354d6c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ ### Internal * Update to Ktor 2.3.4. * Updated to CMake 3.27.7 +* Adding Sync tests via Github Action ## 1.13.1-SNAPSHOT (YYYY-MM-DD) diff --git a/packages/test-sync/src/androidMain/AndroidManifest.xml b/packages/test-sync/src/androidMain/AndroidManifest.xml index ab5ec361ca..ff9d65c6a1 100644 --- a/packages/test-sync/src/androidMain/AndroidManifest.xml +++ b/packages/test-sync/src/androidMain/AndroidManifest.xml @@ -18,9 +18,11 @@ + + + android:usesCleartextTraffic="true">