Skip to content

Commit 060d46c

Browse files
CI: run the Core tests, check vulnerable packages, and fail (not skip) when SQL Server is missing (#138)
* Run the Core tests in CI, check for vulnerable packages, and make a missing SQL Server fail there New tests.yml: the Core suite on net10.0 Release, net11.0 Release and net11.0 Debug on windows-latest, TRX files uploaded, SQL Server LocalDB started first, and a job that fails when `dotnet list package --vulnerable --include-transitive` reports anything. ODX_REQUIRE_SQL=1 (set in that workflow) makes the tests that need LocalDB fail instead of skip when it cannot be reached, also when ODE_TEST_NO_LOCALDB forces a skip. Without it nothing changes. docs.yml: a failing docs sample now fails the run. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> * Simplify two workflow details tests.yml: one concurrency group per ref. A pull request's ref never equals master's, so the separate pull-request group was redundant. docs.yml: drop the samples-failure block from the summary step and the step id it read. A failing sample now simply fails the run and is shown by the failed step itself. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> * Add one test that fails on purpose, to prove CI runs the tests Temporary: the next commit removes it. CI on this commit must be red with CiGuardProof.ThisTestFailsOnPurpose named in the log. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> * Remove the test that failed on purpose Restores the tree of the first commit. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> * Run the docs samples from their project, not from a stale target-framework path The "Run the samples" step ran docsite/samples/bin/Release/net10.0/Samples.dll, but the samples project targets net11.0, so the file was not found. The step was marked continue-on-error, which hid it. Run the project instead, so the target framework comes from the project. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com>
1 parent 44059a5 commit 060d46c

3 files changed

Lines changed: 136 additions & 11 deletions

File tree

‎.github/workflows/docs.yml‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,8 @@ jobs:
4545
- name: Build the samples (a library or sample that does not compile fails the run)
4646
run: dotnet build docsite/samples/Samples.csproj -c Release -p:GeneratePackageOnBuild=false
4747

48-
- name: Run the samples (a failure is reported, not blocking)
49-
id: samples
50-
continue-on-error: true
51-
run: dotnet docsite/samples/bin/Release/net10.0/Samples.dll
48+
- name: Run the samples (a failing sample fails the run)
49+
run: dotnet run --project docsite/samples/Samples.csproj -c Release --no-build
5250

5351
- name: Build the site
5452
run: |
@@ -72,11 +70,6 @@ jobs:
7270
$lines = @()
7371
$lines += '## Docs build'
7472
$lines += ''
75-
if ('${{ steps.samples.outcome }}' -eq 'failure') {
76-
$lines += '> **The samples ran but reported a failure.** The docs may describe behavior that changed. See the "Run the samples" step.'
77-
$lines += ''
78-
Write-Host '::warning title=Docs samples::The samples ran but reported a failure.'
79-
}
8073
if (Test-Path docfx-log.json) {
8174
# Plain text parsing on purpose: a log line that is not valid JSON must not break the summary.
8275
$warnings = @(Get-Content docfx-log.json | Where-Object { $_ -match '"severity":"warning"' } | ForEach-Object {

‎.github/workflows/tests.yml‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
name: Tests
2+
3+
# Runs the Core test suite on every pull request and every push to master, and checks the packages for known vulnerabilities.
4+
# The suite runs on net10.0 and net11.0 in Release, and on net11.0 in Debug (Debug.Assert is live there).
5+
# SQL Server LocalDB is part of the runner image. ODX_REQUIRE_SQL=1 makes the tests that need it fail, instead of skip, when it is missing.
6+
7+
on:
8+
push:
9+
branches: [master]
10+
pull_request:
11+
workflow_dispatch:
12+
13+
permissions:
14+
contents: read
15+
16+
# One group per ref: a pull request's ref (refs/pull/N/merge) never equals master's, so a pull request run can never cancel a master run.
17+
# A newer push to a pull request cancels the older run; master and manual runs queue and are never cancelled.
18+
concurrency:
19+
group: tests-${{ github.ref }}
20+
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
21+
22+
env:
23+
DOTNET_NOLOGO: true
24+
DOTNET_CLI_TELEMETRY_OPTOUT: true
25+
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
26+
27+
jobs:
28+
test:
29+
name: Test ${{ matrix.name }}
30+
runs-on: windows-latest
31+
timeout-minutes: 30
32+
defaults:
33+
run:
34+
shell: pwsh
35+
strategy:
36+
fail-fast: false
37+
matrix:
38+
include:
39+
- name: net10.0-Release
40+
tfm: net10.0
41+
configuration: Release
42+
- name: net11.0-Release
43+
tfm: net11.0
44+
configuration: Release
45+
- name: net11.0-Debug
46+
tfm: net11.0
47+
configuration: Debug
48+
steps:
49+
- uses: actions/checkout@v7
50+
51+
- uses: actions/setup-dotnet@v6
52+
with:
53+
dotnet-version: |
54+
10.0.x
55+
11.0.x
56+
57+
# The tests that need a real SQL Server connect to (localdb)\MSSQLLocalDB. The runner image carries the LocalDB runtime,
58+
# but the automatic instance is created on its first start, which can outlast a connection timeout. Start it here,
59+
# so the first test does not pay for it, and fail now, with a clear message, if the image has no LocalDB.
60+
- name: Start SQL Server LocalDB
61+
run: |
62+
$exe = (Get-Command SqlLocalDB.exe -ErrorAction SilentlyContinue)?.Source
63+
if (-not $exe) {
64+
$exe = Get-ChildItem "$env:ProgramFiles\Microsoft SQL Server\*\Tools\Binn\SqlLocalDB.exe" -ErrorAction SilentlyContinue |
65+
Sort-Object FullName | Select-Object -Last 1 -ExpandProperty FullName
66+
}
67+
if (-not $exe) { throw 'SqlLocalDB.exe was not found on this runner image, so the SQL Server tests cannot run.' }
68+
& $exe start MSSQLLocalDB
69+
if ($LASTEXITCODE -ne 0) { throw "SqlLocalDB start MSSQLLocalDB failed with exit code $LASTEXITCODE" }
70+
& $exe info MSSQLLocalDB
71+
72+
- name: Run the tests (${{ matrix.name }})
73+
env:
74+
ODX_REQUIRE_SQL: 1
75+
run: dotnet test Tests/Open.Database.Extensions.Core.Tests -c ${{ matrix.configuration }} -f ${{ matrix.tfm }} --logger "trx;LogFileName=${{ matrix.name }}.trx" --results-directory TestResults/${{ matrix.name }}
76+
77+
- name: Upload the TRX files
78+
if: always()
79+
uses: actions/upload-artifact@v7
80+
with:
81+
name: trx-${{ matrix.name }}
82+
path: TestResults/${{ matrix.name }}/*.trx
83+
84+
vulnerable-packages:
85+
name: Vulnerable packages
86+
runs-on: windows-latest
87+
timeout-minutes: 15
88+
defaults:
89+
run:
90+
shell: pwsh
91+
steps:
92+
- uses: actions/checkout@v7
93+
94+
- uses: actions/setup-dotnet@v6
95+
with:
96+
dotnet-version: 11.0.x
97+
98+
# `dotnet list package --vulnerable` exits 0 even when it finds something, so read what it prints.
99+
# Each library project, plus the test project, is listed with its transitive packages.
100+
- name: List vulnerable packages (any finding fails the run)
101+
run: |
102+
$projects = @(Get-ChildItem Source -Recurse -Filter *.csproj) + @(Get-Item Tests/Open.Database.Extensions.Core.Tests/*.csproj)
103+
$failures = @()
104+
foreach ($project in $projects) {
105+
$output = (dotnet list $project.FullName package --vulnerable --include-transitive 2>&1 | Out-String)
106+
$exitCode = $LASTEXITCODE
107+
Write-Host $output
108+
if ($exitCode -ne 0) {
109+
$failures += "$($project.Name): dotnet list exited with code $exitCode"
110+
}
111+
elseif ($output -match 'has the following vulnerable packages') {
112+
$failures += "$($project.Name): vulnerable packages found (listed above)"
113+
}
114+
elseif ($output -match 'NU1900') {
115+
$failures += "$($project.Name): the vulnerability data could not be read, so the check proved nothing"
116+
}
117+
}
118+
if ($failures.Count -gt 0) {
119+
foreach ($failure in $failures) { Write-Host "::error title=Vulnerable packages::$failure" }
120+
exit 1
121+
}
122+
"No vulnerable packages in $($projects.Count) project(s)."

‎Tests/Open.Database.Extensions.Core.Tests/TestHelpers/LocalDbFixture.cs‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,12 @@ namespace Open.Database.Extensions.Tests;
77
/// <summary>
88
/// SQL Server LocalDB for the few tests that need a real server (the SQL Server package's typed <c>SqlCommand</c> and <c>SqlTransaction</c> extensions, which fakes cannot stand in for).
99
/// Call <see cref="SkipUnlessAvailable"/> first in such a test: it reports the test as skipped, not as passed or failed, when LocalDB cannot be reached, so the suite stays green on a machine or build agent without it.
10+
/// Where a skip would hide a gap (continuous integration), set the environment variable named by <see cref="RequireVariable"/>: the same tests then fail, with the reason, instead of being skipped.
1011
/// </summary>
1112
/// <remarks>
1213
/// The reachability check runs once per test run. It is the same connection every test uses, so a machine without LocalDB fails fast (the instance does not exist) instead of waiting for a timeout in every test.
1314
/// Setting the environment variable named by <see cref="ForceSkipVariable"/> skips every server test without looking for LocalDB, which is how to see the suite as a machine without LocalDB sees it.
15+
/// <see cref="RequireVariable"/> wins over <see cref="ForceSkipVariable"/>: with both set the server tests fail, so forcing a skip can never turn a required run into a green one.
1416
/// Everything the tests create lives in <c>tempdb</c> under a name made unique by a new <see cref="Guid"/>, and is dropped when its <see cref="Scratch"/> is disposed.
1517
/// </remarks>
1618
[ExcludeFromCodeCoverage]
@@ -19,6 +21,9 @@ internal static class LocalDbFixture
1921
/// <summary>The environment variable that, when it has any value, skips every test that needs LocalDB.</summary>
2022
public const string ForceSkipVariable = "ODE_TEST_NO_LOCALDB";
2123

24+
/// <summary>The environment variable that, when it has any value, makes every test that needs LocalDB fail instead of skip when LocalDB cannot be reached (or when <see cref="ForceSkipVariable"/> is set).</summary>
25+
public const string RequireVariable = "ODX_REQUIRE_SQL";
26+
2227
/// <summary>The default instance every Visual Studio and SQL Server Express LocalDB install creates, working in <c>tempdb</c>.</summary>
2328
public const string ConnectionString = @"Server=(localdb)\MSSQLLocalDB;Database=tempdb;Integrated Security=true;Encrypt=False;Connect Timeout=20";
2429

@@ -42,10 +47,15 @@ internal static class LocalDbFixture
4247
}
4348
}
4449

45-
/// <summary>Skips the calling test (it is reported as skipped) when LocalDB cannot be reached.</summary>
50+
/// <summary>Skips the calling test (it is reported as skipped) when LocalDB cannot be reached, or fails it instead when <see cref="RequireVariable"/> is set.</summary>
4651
public static void SkipUnlessAvailable()
4752
{
48-
if (Unavailable.Value is { } reason) Assert.Skip(reason);
53+
if (Unavailable.Value is not { } reason) return;
54+
55+
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable(RequireVariable)))
56+
Assert.Fail($"{RequireVariable} is set, so a missing SQL Server is a failure, not a skip: {reason}");
57+
58+
Assert.Skip(reason);
4959
}
5060

5161
/// <summary>A new, closed connection to LocalDB's <c>tempdb</c>. The caller disposes it.</summary>

0 commit comments

Comments
 (0)