AB#12334: Prototype APK-compatible test lanes on Helix

build-tools/automation/ApkTestsHelix.proj

@@ -0,0 +1,33 @@
+<Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
+  <PropertyGroup>
+    <Language>msbuild</Language>
+    <IsPosixShell Condition="'$(ApkTestsHelixWorkItemOS)' == 'linux'">true</IsPosixShell>
+    <EnableAzurePipelinesReporter>false</EnableAzurePipelinesReporter>
+    <FailOnTestFailure>true</FailOnTestFailure>
+    <HelixType Condition="'$(HelixType)' == ''">tests/android/apk-tests/</HelixType>
+    <HelixTestRunName Condition="'$(HelixTestRunName)' == ''">Android APK Tests - Helix</HelixTestRunName>
+    <ApkTestsHelixWorkItemTimeout Condition="'$(ApkTestsHelixWorkItemTimeout)' == ''">00:30:00</ApkTestsHelixWorkItemTimeout>
+  </PropertyGroup>
+
+  <Import Project="$(ApkTestsHelixItemsProps)" Condition="Exists('$(ApkTestsHelixItemsProps)')" />
+
+  <ItemGroup>
+    <HelixTargetQueue Include="$(HelixTargetQueueForItem)" Condition="'$(HelixTargetQueueForItem)' != ''">
+      <TestRunName>$(HelixTestRunName)</TestRunName>
+    </HelixTargetQueue>
+    <HelixWorkItem Include="@(_ApkTestHelixWorkItem)">
+      <Timeout>$(ApkTestsHelixWorkItemTimeout)</Timeout>
+    </HelixWorkItem>
+  </ItemGroup>
+
+  <Target Name="_ValidateApkTestsHelixInputs" BeforeTargets="Test">
+    <Error Text="ApkTestsHelixItemsProps must name a generated work-item props file."
+           Condition="'$(ApkTestsHelixItemsProps)' == '' or !Exists('$(ApkTestsHelixItemsProps)')" />
+    <Error Text="HelixTargetQueueForItem is required."
+           Condition="'$(HelixTargetQueueForItem)' == ''" />
+    <Error Text="No APK test Helix work items were generated."
+           Condition="'@(_ApkTestHelixWorkItem->Count())' == '0'" />
+    <Message Text="Submitting @(_ApkTestHelixWorkItem->Count()) APK test work items to $(HelixTargetQueueForItem)."
+             Importance="High" />
+  </Target>
+</Project>

build-tools/automation/apk-tests-helix/ApkTestsHelix.Manifest.targets

@@ -0,0 +1,10 @@
+<Project>
+  <PropertyGroup Condition="'$(_AndroidHelixInstrumentation)' != ''">
+    <AfterGenerateAndroidManifest>$(AfterGenerateAndroidManifest);_InjectApkTestsHelixManifest</AfterGenerateAndroidManifest>
+  </PropertyGroup>
+
+  <Target Name="_InjectApkTestsHelixManifest"
+          Condition="'$(_AndroidHelixInstrumentation)' != '' and '$(AndroidApplication)' == 'true'">
+    <Exec Command="pwsh -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -File &quot;$(ApkTestsHelixManifestScript)&quot; -ManifestPath &quot;$(IntermediateOutputPath)android\AndroidManifest.xml&quot; -PackageName &quot;$(_AndroidHelixPackageName)&quot; -Instrumentation &quot;$(_AndroidHelixInstrumentation)&quot;" />
+  </Target>
+</Project>

build-tools/automation/apk-tests-helix/ApkTestsHelix.Tests.ps1

@@ -0,0 +1,140 @@
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+function Assert-Equal
+{
+	param (
+		[Parameter(Mandatory)]
+		[object] $Expected,
+
+		[Parameter(Mandatory)]
+		[object] $Actual,
+
+		[Parameter(Mandatory)]
+		[string] $Message
+	)
+
+	if ($Expected -ne $Actual) {
+		throw "$Message Expected '$Expected', got '$Actual'."
+	}
+}
+
+$root = Join-Path $PSScriptRoot '.test-output'
+Remove-Item -LiteralPath $root -Recurse -Force -ErrorAction Ignore
+try {
+	$workItem = Join-Path $root 'work-items\sample'
+	$platformTools = Join-Path $root 'platform-tools'
+	New-Item -ItemType Directory -Force -Path $workItem, $platformTools | Out-Null
+	Set-Content -LiteralPath (Join-Path $workItem 'app.apk') -Value 'apk'
+	Set-Content -LiteralPath (Join-Path $platformTools 'adb.exe') -Value 'adb'
+	[pscustomobject] @{
+		name = 'sample'
+		displayName = 'Sample APK Tests'
+		packageName = 'example.tests'
+		instrumentation = 'example.tests.TestInstrumentation'
+		isolatedTest = 'example.tests.FlakyTest'
+	} | ConvertTo-Json | Set-Content -LiteralPath (Join-Path $workItem 'case.json')
+
+	$propsPath = Join-Path $root 'items.props'
+	& (Join-Path $PSScriptRoot 'prepare-apk-test-helix-submission.ps1') `
+		-WorkItemsDirectory (Join-Path $root 'work-items') `
+		-ItemsPropsPath $propsPath `
+		-ResultsDirectory (Join-Path $root 'results') `
+		-PlatformToolsDirectory $platformTools `
+		-TargetMinutes 15
+
+	[xml] $props = Get-Content -LiteralPath $propsPath -Raw
+	$item = $props.Project.ItemGroup._ApkTestHelixWorkItem
+	Assert-Equal 'apk-tests-sample' ([string] $item.Include) 'Work item name should be deterministic.'
+	Assert-Equal 'powershell.exe -NoLogo -NoProfile -ExecutionPolicy Bypass -File run-apk-tests.ps1' ([string] $item.Command) 'Windows preparation should emit a Windows work-item command.'
+	Assert-Equal 'results.trx;console.log;logcat.log;device-state.log;getprop.log;package-state.log;case.json;work-item-error.log' ([string] $item.DownloadFilesFromResults) 'Result files should be downloaded.'
+
+	$script = Get-Content -LiteralPath (Join-Path $workItem 'run-apk-tests.ps1') -Raw
+	if (-not $script.Contains('$packageName = ''example.tests''') -or
+		-not $script.Contains('$instrumentation = ''example.tests.TestInstrumentation''') -or
+		-not $script.Contains("Join-Path `$PSScriptRoot 'platform-tools/adb'") -or
+		-not $script.Contains('INSTRUMENTATION_RESULT: resultsPath=') -or
+		-not $script.Contains('$attempt -le 12') -or
+		-not $script.Contains('$attempt -le 2')) {
+		throw 'Generated APK test script is missing required instrumentation values.'
+	}
+	if (-not (Test-Path -LiteralPath (Join-Path $workItem 'platform-tools\adb.exe') -PathType Leaf)) {
+		throw 'Platform tools were not copied into the work item payload.'
+	}
+	$tokens = $null
+	$errors = $null
+	[Management.Automation.Language.Parser]::ParseFile((Join-Path $workItem 'run-apk-tests.ps1'), [ref] $tokens, [ref] $errors) | Out-Null
+	if ($errors.Count -gt 0) {
+		throw ($errors | ForEach-Object Message | Out-String)
+	}
+	$bashScriptPath = Join-Path $workItem 'run-apk-tests.sh'
+	$bashScript = Get-Content -LiteralPath $bashScriptPath -Raw
+	if (-not $bashScript.Contains('command -v adb') -or
+		-not $bashScript.Contains('INSTRUMENTATION_RESULT: resultsPath=') -or
+		-not $bashScript.Contains("isolated_test='example.tests.FlakyTest'") -or
+		-not $bashScript.Contains('E.register_namespace("", "http://microsoft.com/schemas/VisualStudio/TeamTest/2010")')) {
+		throw 'Generated Linux APK test script is missing required commands.'
+	}
+	if ($bashScript -match '(?m)^\s+(?:E\.register_namespace|tree = E\.parse)') {
+		throw 'Generated TRX merge Python has unexpected top-level indentation.'
+	}
+	if ($bashScript.Contains("`r")) {
+		throw 'Generated Linux APK test script must use LF line endings.'
+	}
+	$git = Get-Command git -ErrorAction Ignore
+	$gitBash = if ($git) {
+		Join-Path (Split-Path (Split-Path $git.Source -Parent) -Parent) 'bin\bash.exe'
+	}
+	if ($gitBash -and (Test-Path -LiteralPath $gitBash -PathType Leaf)) {
+		& $gitBash -n $bashScriptPath
+		if ($LASTEXITCODE -ne 0) {
+			throw "Generated Linux APK test script failed bash syntax validation with exit code $LASTEXITCODE."
+		}
+	}
+
+	$linuxRoot = Join-Path $root 'linux'
+	$linuxWorkItem = Join-Path $linuxRoot 'work-items\sample'
+	New-Item -ItemType Directory -Force -Path $linuxWorkItem | Out-Null
+	Copy-Item -LiteralPath (Join-Path $workItem 'app.apk') -Destination $linuxWorkItem
+	Copy-Item -LiteralPath (Join-Path $workItem 'case.json') -Destination $linuxWorkItem
+	$linuxPropsPath = Join-Path $linuxRoot 'items.props'
+	& (Join-Path $PSScriptRoot 'prepare-apk-test-helix-submission.ps1') `
+		-WorkItemsDirectory (Join-Path $linuxRoot 'work-items') `
+		-ItemsPropsPath $linuxPropsPath `
+		-ResultsDirectory (Join-Path $linuxRoot 'results') `
+		-PlatformToolsDirectory (Join-Path $linuxRoot 'unused-platform-tools') `
+		-WorkItemOS linux `
+		-TargetMinutes 15
+	[xml] $linuxProps = Get-Content -LiteralPath $linuxPropsPath -Raw
+	Assert-Equal 'bash run-apk-tests.sh' ([string] $linuxProps.Project.ItemGroup._ApkTestHelixWorkItem.Command) 'Linux preparation should emit a bash work-item command.'
+	if (Test-Path -LiteralPath (Join-Path $linuxWorkItem 'platform-tools')) {
+		throw 'Linux work items should use the Android queue adb instead of copying Windows platform-tools.'
+	}
+
+	$manifestPath = Join-Path $root 'AndroidManifest.xml'
+	@'
+	<manifest xmlns:android="http://schemas.android.com/apk/res/android" package="example.tests">
+	  <uses-permission android:name="android.permission.INTERNET" />
+	  <application android:name="example.tests.App">
+	    <activity android:name="example.tests.Activity" android:configChanges="keyboardHidden" />
+	  </application>
+	</manifest>
+'@ | Set-Content -LiteralPath $manifestPath
+	& (Join-Path $PSScriptRoot 'inject-apk-test-manifest.ps1') `
+		-ManifestPath $manifestPath `
+		-PackageName 'example.tests' `
+		-Instrumentation 'example.tests.TestInstrumentation'
+
+	[xml] $manifest = Get-Content -LiteralPath $manifestPath -Raw
+	$manager = [Xml.XmlNamespaceManager]::new($manifest.NameTable)
+	$manager.AddNamespace('android', 'http://schemas.android.com/apk/res/android')
+	Assert-Equal 'example.tests.App' $manifest.manifest.application.name 'Manifest injection should preserve the application.'
+	Assert-Equal 'keyboardHidden' $manifest.manifest.application.activity.configChanges 'Manifest injection should preserve activity metadata.'
+	Assert-Equal 1 @($manifest.SelectNodes("/manifest/instrumentation[@android:name='example.tests.TestInstrumentation']", $manager)).Count 'Instrumentation should be added once.'
+	Assert-Equal 1 @($manifest.SelectNodes("/manifest/uses-permission[@android:name='android.permission.ACCESS_LOCAL_NETWORK']", $manager)).Count 'Android 17 local-network permission should be added.'
+	Assert-Equal 1 @($manifest.SelectNodes("/manifest/uses-permission[@android:name='android.permission.NEARBY_WIFI_DEVICES']", $manager)).Count 'Android 16 local-network permission should be added.'
+
+	Write-Host 'ApkTestsHelix tests passed.'
+} finally {
+	Remove-Item -LiteralPath $root -Recurse -Force -ErrorAction Ignore
+}

build-tools/automation/apk-tests-helix/README.md

@@ -0,0 +1,31 @@
+# APK tests in Helix prototype
+
+This guarded prototype builds the existing on-device APK test flavors once on a
+Windows Azure Pipelines agent, then submits one deterministic Helix work item per flavor. Existing
+measurements put every device test invocation below the configurable 15-minute target,
+so splitting an APK's NUnit inventory further would add installation overhead without
+improving the tail.
+
+Each small work item contains a signed APK, Android `platform-tools`, its
+package/instrumentation metadata, and a PowerShell runner, matching the proven
+PR #12020 payload layout.
+The runner installs the APK, invokes `am instrument`, pulls the test-generated TRX,
+captures logcat/device state, and returns failure when instrumentation or any test
+fails.
+
+Preparation preserves the application's normal generated manifest and injects only
+the test instrumentation plus Android 16/17 local-network permissions. The work-item
+runner grants the API-appropriate runtime permission so socket-based tests keep their
+existing semantics on current physical devices. Debug embeds managed assemblies
+because direct APK installation cannot reproduce the existing fast-deployment step;
+that deployment coverage remains in the original lane.
+
+The prototype uses `Ubuntu.2204.Amd64.Android.29.Open`, the public Android emulator
+queue used by dotnet/runtime for x86/x64 Android tests. This matches the existing
+dotnet/android APK lanes' default API 29 emulator instead of distributing the suite
+across heterogeneous physical devices. The runner waits for a delayed device
+assignment and retries instrumentation once while retaining both attempt TRXs. The
+known process-global JNI count checks tracked by dotnet/android#12031 run first in a
+fresh process, while the remainder runs in a second process; the TRXs are merged back
+into the original inventory. The existing macOS emulator lanes remain enabled for
+same-build inventory and outcome comparison.

build-tools/automation/apk-tests-helix/inject-apk-test-manifest.ps1

@@ -0,0 +1,64 @@
+[CmdletBinding()]
+param (
+	[Parameter(Mandatory)]
+	[string] $ManifestPath,
+
+	[Parameter(Mandatory)]
+	[string] $PackageName,
+
+	[Parameter(Mandatory)]
+	[string] $Instrumentation
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+
+$manifestFullPath = [IO.Path]::GetFullPath($ManifestPath)
+if (-not (Test-Path -LiteralPath $manifestFullPath -PathType Leaf)) {
+	throw "Android manifest '$manifestFullPath' does not exist."
+}
+
+[xml] $document = Get-Content -LiteralPath $manifestFullPath -Raw
+$manifest = $document.DocumentElement
+if (-not $manifest -or $manifest.LocalName -ne 'manifest') {
+	throw "Android manifest '$manifestFullPath' has no manifest root element."
+}
+
+$androidNamespace = 'http://schemas.android.com/apk/res/android'
+$namespaceManager = [Xml.XmlNamespaceManager]::new($document.NameTable)
+$namespaceManager.AddNamespace('android', $androidNamespace)
+
+foreach ($permission in @(
+	'android.permission.ACCESS_LOCAL_NETWORK'
+	'android.permission.NEARBY_WIFI_DEVICES'
+)) {
+	$existing = $manifest.SelectSingleNode("uses-permission[@android:name='$permission']", $namespaceManager)
+	if (-not $existing) {
+		$element = $document.CreateElement('uses-permission')
+		$element.SetAttribute('name', $androidNamespace, $permission) | Out-Null
+		$manifest.PrependChild($element) | Out-Null
+	}
+}
+
+$instrumentationElement = $manifest.SelectSingleNode("instrumentation[@android:name='$Instrumentation']", $namespaceManager)
+if (-not $instrumentationElement) {
+	$instrumentationElement = $document.CreateElement('instrumentation')
+	$instrumentationElement.SetAttribute('name', $androidNamespace, $Instrumentation) | Out-Null
+	$instrumentationElement.SetAttribute('targetPackage', $androidNamespace, $PackageName) | Out-Null
+	$manifest.AppendChild($instrumentationElement) | Out-Null
+} else {
+	$targetPackage = $instrumentationElement.GetAttribute('targetPackage', $androidNamespace)
+	if ($targetPackage -ne $PackageName) {
+		throw "Instrumentation '$Instrumentation' targets '$targetPackage' instead of '$PackageName'."
+	}
+}
+
+$settings = [Xml.XmlWriterSettings]::new()
+$settings.Encoding = [Text.UTF8Encoding]::new($false)
+$settings.Indent = $true
+$writer = [Xml.XmlWriter]::Create($manifestFullPath, $settings)
+try {
+	$document.Save($writer)
+} finally {
+	$writer.Dispose()
+}