diff --git a/.github/workflows/android-smoke-test-wrapper.yml b/.github/workflows/android-smoke-test-wrapper.yml deleted file mode 100644 index 0ce48596c..000000000 --- a/.github/workflows/android-smoke-test-wrapper.yml +++ /dev/null @@ -1,35 +0,0 @@ -on: - workflow_call: - inputs: - unity-version: - required: true - type: string - api-level: - required: true - type: string - -jobs: - try-1: - uses: ./.github/workflows/android-smoke-test.yml - with: - unity-version: ${{ inputs.unity-version }} - api-level: ${{ inputs.api-level }} - try: 1 - - try-2: - needs: [try-1] - if: ${{ needs.try-1.result == 'failure' }} - uses: ./.github/workflows/android-smoke-test.yml - with: - unity-version: ${{ inputs.unity-version }} - api-level: ${{ inputs.api-level }} - try: 2 - - try-3: - needs: [try-2] - if: ${{ needs.try-2.result == 'failure' }} - uses: ./.github/workflows/android-smoke-test.yml - with: - unity-version: ${{ inputs.unity-version }} - api-level: ${{ inputs.api-level }} - try: 3 diff --git a/.github/workflows/android-smoke-test.yml b/.github/workflows/android-smoke-test.yml index e8cc4b9a7..37a26f9bb 100644 --- a/.github/workflows/android-smoke-test.yml +++ b/.github/workflows/android-smoke-test.yml @@ -7,19 +7,16 @@ on: api-level: required: true type: string - try: - required: true - type: number # Map the workflow outputs to job outputs outputs: - outcome: - description: - value: ${{ jobs.run.outputs.outcome }} + status: + description: "Smoke test status" + value: ${{ jobs.run.outputs.status }} jobs: run: - name: try-${{ inputs.try }} - runs-on: macos-latest + name: Android Smoke Test + runs-on: ubuntu-latest env: ARTIFACTS_PATH: samples/IntegrationTest/test-artifacts/ HOMEBREW_NO_INSTALL_CLEANUP: 1 @@ -28,17 +25,24 @@ jobs: shell: pwsh # Map the job outputs to step outputs outputs: - outcome: ${{ steps.smoke-test.outcome }} + status: ${{ steps.smoke-test.outputs.status }} steps: - name: Checkout - uses: actions/checkout@v3 - + uses: actions/checkout@v4 + - name: Download test app artifact uses: actions/download-artifact@v4 with: name: testapp-Android-compiled-${{ inputs.unity-version }} path: samples/IntegrationTest/Build + # See https://github.blog/changelog/2023-02-23-hardware-accelerated-android-virtualization-on-actions-windows-and-linux-larger-hosted-runners/ + - name: Enable KVM group perms + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + # outputs variables: api-level, label, target - name: Configure Android Settings id: config @@ -64,24 +68,23 @@ jobs: "target=$target" >> $env:GITHUB_OUTPUT "api-level=$apiLevel" >> $env:GITHUB_OUTPUT "label=$($label ?? $apiLevel)" >> $env:GITHUB_OUTPUT - - - name: Android API ${{ steps.config.outputs.label }} emulator setup + Smoke test - uses: reactivecircus/android-emulator-runner@d94c3fbe4fe6a29e4a5ba47c12fb47677c73656b # pin@v2.28.0 + + - name: Run Android Smoke Tests + uses: reactivecircus/android-emulator-runner@d94c3fbe4fe6a29e4a5ba47c12fb47677c73656b # pin@v2.33.0 id: smoke-test - timeout-minutes: 40 - continue-on-error: true + timeout-minutes: 30 with: api-level: ${{ steps.config.outputs.api-level }} target: ${{ steps.config.outputs.target }} force-avd-creation: false - ram-size: 2048M - arch: x86_64 - cores: 3 - emulator-boot-timeout: 1800 - disk-size: 4096M # Some runs have out of storage error when installing the smoke test. - emulator-options: -no-snapshot-save -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none -accel on + emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none disable-animations: true - script: pwsh ./scripts/smoke-test-android.ps1 -IsIntegrationTest -WarnIfFlaky + arch: x86_64 + script: | + adb wait-for-device + adb shell input keyevent 82 + adb devices -l + pwsh ./scripts/smoke-test-android.ps1 -IsIntegrationTest -WarnIfFlaky - name: Upload artifacts on failure if: ${{ failure() }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce07b1837..a99bd6502 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -470,26 +470,15 @@ jobs: if: ${{ !startsWith(github.ref, 'refs/heads/release/') }} needs: [mobile-smoke-test-compile] name: ${{ matrix.unity-version }} Android ${{ matrix.api-level }} Run Smoke Test - uses: ./.github/workflows/android-smoke-test-wrapper.yml + uses: ./.github/workflows/android-smoke-test.yml with: unity-version: ${{ matrix.unity-version }} api-level: ${{ matrix.api-level }} strategy: fail-fast: false matrix: - api-level: [27, 28, 29, 30, 31] # last updated October 2022 + api-level: [27, 31, 34] # last updated December 2024 unity-version: ["2019", "2022", "6000"] - include: - # API 21 is barely used but let's check it as the minimum supported version for now. - - api-level: 21 - unity-version: "2019" - exclude: - # Seems like there's an error in Unity with Android API 30 - disabling. - # https://github.com/getsentry/sentry-unity/issues/719#issuecomment-1129129952 - - api-level: 30 - unity-version: "2021" - - api-level: 30 - unity-version: "2022" mobile-smoke-test-compile: if: ${{ !startsWith(github.ref, 'refs/heads/release/') }} diff --git a/scripts/smoke-test-android.ps1 b/scripts/smoke-test-android.ps1 index 5dccfd0f1..686060db3 100644 --- a/scripts/smoke-test-android.ps1 +++ b/scripts/smoke-test-android.ps1 @@ -14,17 +14,19 @@ if (-not $Global:NewProjectPathCache) . $PSScriptRoot/../test/Scripts.Integration.Test/common.ps1 # GITHUB_WORKSPACE is the root folder where the project is stored. -Write-Host "#################################################" -Write-Host "# ANDROID #" -Write-Host "# VALIDATOR #" -Write-Host "# SCRIPT #" -Write-Host "#################################################" +Write-Host "#####################################################" +Write-Host "# ___ __ __ ___ _ _____ _____ ___ ____ _____ #" +Write-Host "# / __| \/ |/ _ \| |/ / __|_ _| __|/ __|_ _| #" +Write-Host "# \__ \ |\/| | (_) | ' <| _| | | | _| \__ \ | | #" +Write-Host "# |___/_| |_|\___/|_|\_\___| |_| |___|___/ |_| #" +Write-Host "# #" +Write-Host "#####################################################" if ($IsIntegrationTest) { $BuildDir = $(GetNewProjectBuildPath) $ApkFileName = "test.apk" - $ProcessName = "com.DefaultCompany.IntegrationTest" + $ProcessName = "com.DefaultCompany.$(GetNewProjectName)" if ($Action -eq "Build") { @@ -62,6 +64,7 @@ else $ProcessName = "io.sentry.samples.unityofbugs" } $TestActivityName = "$ProcessName/com.unity3d.player.UnityPlayerActivity" +$FallBackTestActivityName = "$ProcessName/com.unity3d.player.UnityPlayerGameActivity" $_ArtifactsPath = ((Test-Path env:ARTIFACTS_PATH) ? $env:ARTIFACTS_PATH : "./$BuildDir/../test-artifacts/") ` + $(Get-Date -Format "HHmmss") @@ -74,12 +77,6 @@ function ArtifactsPath $_ArtifactsPath.Replace('\', '/') } -if (Test-Path env:CI) -{ - # Take Screenshot of VM to verify emulator start - screencapture "$(ArtifactsPath)/host-screenshot.jpg" -} - function TakeScreenshot([string] $deviceId) { $file = "/data/local/tmp/screen$(Get-Date -Format "HHmmss").png" @@ -120,15 +117,32 @@ function LogCat([string] $deviceId, [string] $appPID) function PidOf([string] $deviceId, [string] $processName) { - if ($deviceApi -eq "21") + $startTime = Get-Date + $timeout = New-TimeSpan -Seconds 60 + + while ((Get-Date) - $startTime -lt $timeout) { - # `pidof` doesn't exist - take second column from the `ps` output for the given process instead. - (adb -s $deviceId shell "ps | grep '$processName'") -Split " +" | Select-Object -Skip 1 -First 1 - } - else - { - adb -s $deviceId shell pidof $processName + if ($deviceApi -eq "21") + { + # `pidof` doesn't exist - take second column from the `ps` output for the given process instead. + $processId = (adb -s $deviceId shell "ps | grep '$processName'") -Split " +" | Select-Object -Skip 1 -First 1 + } + else + { + $processId = adb -s $deviceId shell pidof $processName + } + + if (-not [string]::IsNullOrWhiteSpace($processId)) + { + return $processId + } + + Write-Host "Process '$processName' not found, retrying in 2 seconds..." + Start-Sleep -Seconds 2 } + + Write-Host "Could not find PID for process '$processName' after 60 seconds" -ForegroundColor Red + return $null } function OnError([string] $deviceId, [string] $deviceApi, [string] $appPID) @@ -144,128 +158,7 @@ function OnError([string] $deviceId, [string] $deviceApi, [string] $appPID) TakeScreenshot $device } - -function CloseSystemAlert([string] $deviceId, [string] $deviceApi, [string] $alert) -{ - if ("$alert" -ne "") - { - $splitXml = $alert -split ".+)\`" resource-id") - $alertTitle = $titleRegex.Groups["text"].Value - } - elseif ($iterator.Contains("Button")) - { - $buttonRegex = [regex]::Match($iterator, "text=\`"(?.+)\`" resource-id.* bounds=\`"\[(?\d+),(?\d+)\]\[(?\d+),(?\d+)\]\`"") - if ($null -eq $alertOption1Label) - { - $alertOption1Label = $buttonRegex.Groups["text"].Value - $alertOption1Coord = ($buttonRegex.Groups["horStart"].Value, $buttonRegex.Groups["verStart"].Value, $buttonRegex.Groups["horEnd"].Value, $buttonRegex.Groups["verEnd"].Value) - } - else - { - $alertOption2Label = $buttonRegex.Groups["text"].Value - $alertOption2Coord = ($buttonRegex.Groups["horStart"].Value, $buttonRegex.Groups["verStart"].Value, $buttonRegex.Groups["horEnd"].Value, $buttonRegex.Groups["verEnd"].Value) - } - } - } - - if ($null -ne $alertTitle) - { - Write-Warning "Found Alert on Screen, TITLE: $alertTitle `n Options: `n $alertOption1Label at $alertOption1Coord `n $alertOption2Label at $alertOption2Coord " - - if ($null -eq $alertOption2Label) - { - $tapX = [int]([int]$alertOption1Coord[0] + [int]$alertOption1Coord[2] ) / 2 - $tapY = [int]([int]$alertOption1Coord[1] + [int]$alertOption1Coord[3] ) / 2 - $tapLabel = $alertOption1Label - } - else - { - $tapX = [int]([int]$alertOption2Coord[0] + [int]$alertOption2Coord[2] ) / 2 - $tapY = [int]([int]$alertOption2Coord[1] + [int]$alertOption2Coord[3] ) / 2 - $tapLabel = $alertOption2Label - } - Write-Host "Tapping on $tapLabel at [$tapX, $tapY]" - adb -s $deviceId shell input tap $tapX $tapY - } - } - else - { - # Fallback to the old method of closing Alerts. (Android API 21 to 27) - Write-Warning "Active system alert found on $deviceId (API $deviceApi). Closing it. The alert was: '$alert'." - if ($deviceApi -eq "21") - { - Write-Warning "Issuing ENTER command twice to close the current window." - # sends "enter" - the first one focus the OK button, the second one taps it - adb -s $deviceId shell input keyevent 66 - adb -s $deviceId shell input keyevent 66 - } - else - { - # sends "back" action - Write-Warning "Issuing BACK command to close the current window." - adb -s $deviceId shell input keyevent 4 - } - } - } -} - -function CheckAndCloseActiveSystemAlerts([string] $deviceId, [string] $deviceApi) -{ - $uiInfoXml = GetDeviceUiLog $deviceId $deviceApi - if ($deviceApi -eq "21") - { - CloseSystemAlert $deviceId $deviceApi ($uiInfoXml | Select-String "has stopped") - } - else - { - CloseSystemAlert $deviceId $deviceApi ($uiInfoXml | Select-String "android:id/alertTitle|has stopped|Close app") - } -} - -function ExitNow([string] $status, [string] $message) -{ - if (Test-Path env:GITHUB_OUTPUT) - { - Write-Host "Writing 'status=$status' to env:GITHUB_OUTPUT: ${env:GITHUB_OUTPUT}" - "status=$status" >> $env:GITHUB_OUTPUT - } - else - { - Write-Host "status=$status" - } - - if ($status -ieq "success") - { - Write-Host $message -ForegroundColor Green - } - elseif ($status -ieq "flaky" -and $WarnIfFlaky) - { - Write-Warning $message - } - else - { - Write-Error $message - exit 1 # just in case error handling is overriden - } - exit 0 -} - -# Filter device List +# Filter device list $RawAdbDeviceList = adb devices $DeviceList = @() @@ -276,11 +169,12 @@ foreach ($device in $RawAdbDeviceList) $DeviceList += $device.Replace("device", '').Trim() } } -$DeviceCount = $DeviceList.Count +$DeviceCount = $DeviceList.Count If ($DeviceCount -eq 0) { - ExitNow "failed" "It seems like no devices were found $RawAdbDeviceList" + Write-Error "It seems like no devices were found $RawAdbDeviceList" + exit 1 } Else { @@ -290,179 +184,288 @@ Else # Check if APK was built. If (-not (Test-Path -Path "$BuildDir/$ApkFileName" )) { - ExitNow "failed" "Expected APK on $BuildDir/$ApkFileName but it was not found." + Write-Error "Expected APK on $BuildDir/$ApkFileName but it was not found." + exit 1 } -# Test -foreach ($device in $DeviceList) + +### START TEST + +# Pick the first device available +$device = $DeviceList[0] +adb -s $device logcat -c + +$deviceApi = "$(adb -s $device shell getprop ro.build.version.sdk)".Trim() +$deviceSdk = "$(adb -s $device shell getprop ro.build.version.release)".Trim() +Write-Host "`nChecking device $device with SDK '$deviceSdk' and API '$deviceApi'" + +# Uninstall previous installation +$stdout = adb -s $device shell "pm list packages -f" +if ($null -ne ($stdout | Select-String $ProcessName)) { - adb -s $device logcat -c + Write-Host "Uninstalling previous $ProcessName." + $stdout = adb -s $device uninstall $ProcessName +} - $deviceApi = "$(adb -s $device shell getprop ro.build.version.sdk)".Trim() - $deviceSdk = "$(adb -s $device shell getprop ro.build.version.release)".Trim() - Write-Host "`nChecking device $device with SDK '$deviceSdk' and API '$deviceApi'" +# Move device to home screen +$stdout = adb -s $device shell input keyevent KEYCODE_HOME - if (Test-Path env:CI) +# Install the test app +$adbInstallRetry = 5 +do +{ + Write-Host "Installing test app" + $stdout = (adb -s $device install -r $BuildDir/$ApkFileName 2>&1) + + if ($stdout.Contains("Broken pipe")) { - # Take Screenshot of the device to verify emulator start - TakeScreenshot $device + Write-Warning "Failed to comunicate with the Device, retrying..." + Start-Sleep 3 + $adbInstallRetry-- } +} while ($adbInstallRetry -gt 1 -and $stdout.Contains("Broken pipe")) - $stdout = adb -s $device shell "pm list packages -f" - if ($null -ne ($stdout | Select-String $ProcessName)) - { - Write-Host "Removing previous APP." - $stdout = adb -s $device uninstall $ProcessName +# Validate the installation +If ($stdout -contains "Success") +{ + Write-Host "Successfully installed APK" +} +else +{ + OnError $device $deviceApi + Write-Error "Failed to install APK: $stdout." + return 1 +} + +function ProcessNewLogs([array]$newLogs, [ref]$lastLogCount, [array]$logCache) { + if ($newLogs) { + $currentLogs = @($newLogs) # Force array creation even for single line + if ($currentLogs.Count -gt $lastLogCount.Value) { + $newLines = $currentLogs[$lastLogCount.Value..($currentLogs.Count-1)] + $lastLogCount.Value = $currentLogs.Count + + if ($newLines) { + $logCache += $newLines + } + } } + return $logCache +} + +function RunTest([string] $Name, [string] $SuccessString, [string] $FailureString) +{ + Write-Host "::group::Test: '$name'" + + Write-Host "Clearing logcat from '$device'" + adb -s $device logcat -c + + $activityName = $TestActivityName - # Move device to home screen - $stdout = adb -s $device shell input keyevent KEYCODE_HOME + Write-Host "Setting configuration" - $adbInstallRetry = 5 - do + # Mark the full-screen notification as acknowledged + adb -s $device shell "settings put secure immersive_mode_confirmations confirmed" + adb -s $device shell "input keyevent KEYCODE_HOME" + + Write-Host "Starting app '$activityName'" + + # Start the adb command as a background job with a 30-second timeout + $job = Start-Job -ScriptBlock { + param($device, $activityName, $Name) + & adb -s $device shell am start -n $activityName -e test $Name -W 2>&1 + } -ArgumentList $device, $activityName, $Name + + # Wait for the job to complete or timeout after 30 seconds + $completed = Wait-Job $job -Timeout 60 + if ($null -eq $completed) { + Stop-Job $job + Remove-Job $job -Force + Write-Host "Activity start timed out after 60 seconds" + return $false + } + + $output = Receive-Job $job + Remove-Job $job + + Write-Host "Checking if activity started" + + # Check if the activity failed to start + if ($output -match "Error type 3" -or $output -match "Activity class \{$activityName\} does not exist.") { - Write-Host "Installing test app..." - $stdout = (adb -s $device install -r $BuildDir/$ApkFileName 2>&1) + $activityName = $FallBackTestActivityName + Write-Host "Trying fallback activity $activityName" - if ($stdout.Contains("Broken pipe")) + $output = & adb -s $device shell am start -n $activityName -e test $Name -W 2>&1 + + # Check if the fallback activity failed to start + if ($output -match "Error type 3" -or $output -match "Activity class \{$activityName\} does not exist.") { - Write-Warning "Failed to comunicate with the Device, retrying..." - Start-Sleep 3 - $adbInstallRetry-- + Write-Host "Activity does not exist" + return $false } - } while ($adbInstallRetry -gt 1 -and $stdout.Contains("Broken pipe")) + } + + Write-Host "Activity started successfully" - If ($stdout -notcontains "Success") + $appPID = PidOf $device $ProcessName + if ($null -eq $appPID) { - OnError $device $deviceApi - ExitNow "failed" "Failed to Install APK: $stdout." + Write-Host "::endgroup::" + Write-Host "Retrieving process ID failed. Skipping test." -ForegroundColor Red + return $false } - function RunTest([string] $Name, [string] $SuccessString, [string] $FailureString, [switch] $PreserveLogcat) + Write-Host "Retrieved ID for '$ProcessName': $appPID" + + Write-Host "Waiting for tests to run..." + + $processFinished = $false + $logCache = @() + $startTime = Get-Date + $timeout = New-TimeSpan -Seconds 500 + $lastLogCount = 0 + + # Wait for the tests to run and the game process to complete + while ((Get-Date) - $startTime -lt $timeout) { - Write-Host "::group::Test: '$name'" + $newLogs = adb -s $device logcat -d --pid=$appPID + $logCache = ProcessNewLogs -newLogs $newLogs -lastLogCount ([ref]$lastLogCount) -logCache $logCache - if (!$PreserveLogcat) + # The SmokeTester logs "SmokeTester is quitting." in OnApplicationQuit() to reliably inform when tests finish running. + # For crash tests, we expect to see a native crash log "terminating with uncaught exception of type char const*". + if (($newLogs | Select-String "SmokeTester is quitting.") -or ($newLogs | Select-String "terminating with uncaught exception of type char const*")) { - Write-Host "Clearing logcat from $device." - adb -s $device logcat -c + Write-Host "Process finished marker detected. Finish waiting for tests to run." + $processFinished = $true + break } - adb -s $device shell am start -n $TestActivityName -e test $Name - #despite calling start, the app might not be started yet. + Start-Sleep -Seconds 1 + } - $timedOut = $true - $appPID = $null - $stopwatch = [System.Diagnostics.Stopwatch]::StartNew() - While ($stopwatch.Elapsed.TotalSeconds -lt 60) - { - # Check if the app started - it's not absolutely necessary to get the PID, just useful to achieve good log output. - if ($null -eq $appPID) - { - $appPID = PidOf $device $ProcessName - if ($null -eq $appPID) - { - if ($stopwatch.Elapsed.TotalSeconds % 10 -eq 0) - { - Write-Host "Waiting Process on $device to start, time elapsed already: $($stopwatch.Elapsed.ToString('hh\:mm\:ss\.fff'))" - } - # No sleep here or we may miss the start. While it's not critical, it's useful to get the PID. - continue - } - } + if ($processFinished) + { + Write-Host "'$Name' test finished running." + } + else + { + Write-Host "'$Name' tests timed out. See logcat for more details." + } - $isRunning = $null -ne (PidOf $device $ProcessName) - If ($isRunning) - { - Write-Host "Waiting Process $appPID on $device to complete, time elapsed already: $($stopwatch.Elapsed.ToString('hh\:mm\:ss\.fff'))" - Start-Sleep -Seconds 1 - CheckAndCloseActiveSystemAlerts $device $deviceApi - } - else - { - $timedOut = $false - break - } - } + Write-Host "::endgroup::" - if ("$SuccessString" -eq "") - { - $SuccessString = "$($Name.ToUpper()) TEST: PASS" - } + # Fetch the latest logs from the device + $logCache = ProcessNewLogs -newLogs $newLogs -lastLogCount ([ref]$lastLogCount) -logCache $logCache - if ("$FailureString" -eq "") - { - $FailureString = "$($Name.ToUpper()) TEST: FAIL" - } + Write-Host "::group::logcat" + $logCache | ForEach-Object { Write-Host $_ } + Write-Host "::endgroup::" - $LogcatCache = LogCat $device $appPID - $lineWithSuccess = $LogcatCache | Select-String $SuccessString - $lineWithFailure = $LogcatCache | Select-String $FailureString + $lineWithSuccess = $logCache | Select-String $SuccessString + $lineWithFailure = $logCache | Select-String $FailureString - if ($lineWithFailure -eq $null) - { - $lineWithFailure = $LogcatCache | Select-String "Error: Activity class .* does not exist." - } + if ($null -ne $lineWithSuccess) + { + Write-Host "'$Name' test passed." -ForegroundColor Green + return $true + } + elseif ($null -ne $lineWithFailure) + { + Write-Host "'$Name' test failed. See logcat for more details." -ForegroundColor Red + return $false + } + + Write-Host "'$Name' test execution failed." -ForegroundColor Red + return $false +} - If ($lineWithFailure -ne $null) - { - Write-Host "::endgroup::" - OnError $device $deviceApi $appPID - ExitNow "failed" "$Name test: FAIL - $lineWithFailure" - } - elseif ($lineWithSuccess -ne $null) - { - Write-Host "$lineWithSuccess" - Write-Host "$Name test: PASS" -ForegroundColor Green - Write-Host "::endgroup::" - } - ElseIf (($LogcatCache | Select-String 'CRASH :')) - { - Write-Host "::endgroup::" - OnError $device $deviceApi $appPID - ExitNow "crashed" "$name test app has crashed." - } - ElseIf (($LogcatCache | Select-String 'Unity : Timeout while trying detaching primary window.')) +function RunTestWithRetry([string] $Name, [string] $SuccessString, [string] $FailureString, [int] $MaxRetries = 3) +{ + for ($retryCount = 0; $retryCount -lt $MaxRetries; $retryCount++) + { + if ($retryCount -gt 0) { - Write-Host "::endgroup::" - OnError $device $deviceApi $appPID - ExitNow "flaky" "$name test was flaky, unity failed to initialize." + Write-Host "Retry attempt $retryCount for test '$Name'" + Start-Sleep -Seconds 2 # Brief pause between retries } - ElseIf ($timedOut) + + Write-Host "Running test attempt $($retryCount + 1)/$MaxRetries" + $result = RunTest -Name $Name -SuccessString $SuccessString -FailureString $FailureString + + if ($result) { - Write-Host "::endgroup::" - Write-Host "::group::Processes running on device" - adb -s $device shell ps - Write-Host "::endgroup::" - OnError $device $deviceApi $appPID - ExitNow "timeout" "$name test Timeout, see Logcat info for more info." + Write-Host "'$Name' test passed on attempt $($retryCount + 1)." -ForegroundColor Green + return $true } - Else + + if ($retryCount + 1 -lt $MaxRetries) { - Write-Host "::endgroup::" - OnError $device $deviceApi $appPID - ExitNow "failed" "$name test: failed - process completed but $Name test was not signaled." + Write-Host "'$Name' test failed. Retrying..." -ForegroundColor Yellow + continue } + + Write-Host "'$Name' test failed after $MaxRetries attempts." -ForegroundColor Red + return $false } + + return $false +} - RunTest -Name "smoke" - RunTest -Name "hasnt-crashed" +$results = @{ + smokeTestPassed = $false + hasntCrashedTestPassed = $false + crashTestPassed = $false + hasCrashTestPassed = $false +} - try - { - # Note: mobile apps post the crash on the second app launch, so we must run both as part of the "CrashTestWithServer" - CrashTestWithServer -SuccessString "POST /api/12345/envelope/ HTTP/1.1`" 200 -b'1f8b08000000000000" -CrashTestCallback { - RunTest -Name "crash" -SuccessString "CRASH TEST: Issuing a native crash" -FailureString "CRASH TEST: FAIL" - RunTest -Name "has-crashed" -PreserveLogcat - } - } - catch - { - Write-Warning "Caught exception: $_" - Write-Host $_.ScriptStackTrace - OnError $device $deviceApi - ExitNow "failed" $_; +$results.smoketestPassed = RunTestWithRetry -Name "smoke" -SuccessString "SMOKE TEST: PASS" -FailureString "SMOKE TEST: FAIL" -MaxRetries 3 +$results.hasntCrashedTestPassed = RunTestWithRetry -Name "hasnt-crashed" -SuccessString "HASNT-CRASHED TEST: PASS" -FailureString "HASNT-CRASHED TEST: FAIL" -MaxRetries 3 + +try +{ + CrashTestWithServer -SuccessString "POST /api/12345/envelope/ HTTP/1.1`" 200 -b'1f8b08000000000000" -CrashTestCallback { + $results.crashTestPassed = RunTestWithRetry -Name "crash" -SuccessString "CRASH TEST: Issuing a native crash" -FailureString "CRASH TEST: FAIL" -MaxRetries 3 + $results.hasCrashTestPassed = RunTestWithRetry -Name "has-crashed" -SuccessString "HAS-CRASHED TEST: PASS" -FailureString "HAS-CRASHED TEST: FAIL" -MaxRetries 3 } } +catch +{ + Write-Host "Caught exception: $_" + Write-Host $_.ScriptStackTrace + OnError $device $deviceApi + return $false +} + +$failed = $false + +if (-not $results.smoketestPassed) +{ + Write-Host "Smoke test failed" + $failed = $true +} + +if (-not $results.hasntCrashedTestPassed) +{ + Write-Host "HasntCrashed test failed" + $failed = $true +} + +if (-not $results.crashTestPassed) +{ + Write-Host "Crash test failed" + $failed = $true +} + +if (-not $results.hasCrashTestPassed) +{ + Write-Host "HasCrashed test failed" + $failed = $true +} + +if ($failed) +{ + exit 1 +} -ExitNow "success" "Tests completed successfully." +Write-Host "All tests passed" -ForegroundColor Green +exit 0 diff --git a/test/Scripts.Integration.Test/Scripts/SmokeTester.cs b/test/Scripts.Integration.Test/Scripts/SmokeTester.cs index a6bedf99b..edd0a4430 100644 --- a/test/Scripts.Integration.Test/Scripts/SmokeTester.cs +++ b/test/Scripts.Integration.Test/Scripts/SmokeTester.cs @@ -20,6 +20,11 @@ public class SmokeTester : MonoBehaviour private void Awake() { Debug.Log("SmokeTester, awake!"); + Application.quitting += () => + { + // We're using this in the smoke-test-android.ps1 script to reliably detect when the tests have finished running. + Debug.Log("SmokeTester is quitting."); + }; } public void Start() @@ -156,6 +161,8 @@ public static void SmokeTest() t.ExpectMessage(currentMessage, "'filename':'screenshot.jpg','attachment_type':'event.attachment'"); t.ExpectMessageNot(currentMessage, "'length':0"); + Debug.Log("SMOKE TEST: Finished checking messages."); + t.Pass(); } catch (Exception ex) diff --git a/test/Scripts.Integration.Test/common.ps1 b/test/Scripts.Integration.Test/common.ps1 index 5d50641c1..164e7a9dd 100644 --- a/test/Scripts.Integration.Test/common.ps1 +++ b/test/Scripts.Integration.Test/common.ps1 @@ -79,6 +79,8 @@ function CrashTestWithServer([ScriptBlock] $CrashTestCallback, [string] $Success throw "SuccessString cannot be empty" } + Write-Host "Running crash test with server" -ForegroundColor Yellow + # You can increase this to retry multiple times. Seems a bit flaky at the moment in CI. if ($null -eq $env:CI) { diff --git a/test/Scripts.Integration.Test/integration-test.ps1 b/test/Scripts.Integration.Test/integration-test.ps1 index 77b5e6fc7..b0f80b430 100644 --- a/test/Scripts.Integration.Test/integration-test.ps1 +++ b/test/Scripts.Integration.Test/integration-test.ps1 @@ -56,7 +56,8 @@ If ($Clean) } } -If (-not(Test-Path -Path $PackageReleaseOutput) -Or $Repack) +# Repackaging the SDK +If ($Repack -Or -not(Test-Path -Path $PackageReleaseOutput)) { dotnet build Write-Host "Creating Package"