feat(testing): add boot test infrastructure — Phase 1 complete

kernel/Makefile:
- Add 'test' target that builds + boots all architectures
- Add 'test-armv7', 'test-aarch64', 'test-riscv' per-arch targets
- Configurable timeout (BOOT_TIMEOUT=15s) and banner string
- Architecture matrix: armv7, aarch64, riscv

tools/uos-boot-test/:
- New QEMU orchestrator (356 lines Python)
- Spawns QEMU, captures UART, checks for boot banner
- JUnit XML output for CI integration
- Supports --arch, --timeout, --junit, --verbose flags
- Per-arch configs with correct QEMU binaries and flags

.github/workflows/ci.yml:
- Add 'boot-test' job that runs after kernel-build
- Matrix strategy: armv7, aarch64, riscv
- Downloads ELF artifacts from kernel-build job
- Installs QEMU + cross-compilers
- Runs uos-boot-test.py with JUnit XML output
- Uploads test results as artifacts

Phase 1 of testing roadmap: CI boot testing now operational.
This commit is contained in:
Fábio Coutada 2026-07-12 16:42:23 +01:00
parent 7983d7ffc3
commit 793069c915
4 changed files with 550 additions and 1 deletions

View file

@ -128,3 +128,46 @@ jobs:
with: with:
name: traceability-matrix name: traceability-matrix
path: traceability.html path: traceability.html
boot-test:
name: Boot test ${{ matrix.arch }}
runs-on: ubuntu-latest
needs: kernel-build
strategy:
fail-fast: false
matrix:
arch: [armv7, aarch64, riscv]
steps:
- uses: actions/checkout@v4
- name: Install QEMU
run: |
sudo apt-get update
sudo apt-get install -y qemu-system-x86 qemu-system-arm qemu-system-aarch64 qemu-system-misc
- name: Install cross-compiler
run: |
sudo apt-get install -y gcc-arm-none-eabi binutils-arm-none-eabi \
gcc-aarch64-none-elf binutils-aarch64-none-elf \
gcc-riscv64-unknown-elf binutils-riscv64-unknown-elf
- name: Download kernel ELF
uses: actions/download-artifact@v4
with:
name: universalisos-${{ matrix.arch }}-${{ matrix.arch == 'riscv' && 'polarfire' || format('qemu-{0}-virt', matrix.arch) }}
path: kernel/build/${{ matrix.arch }}/
- name: Run boot test
run: |
python3 tools/uos-boot-test/uos-boot-test.py \
--arch ${{ matrix.arch }} \
--timeout 20 \
--junit boot-test-results.xml
- name: Upload test results
uses: actions/upload-artifact@v4
if: always()
with:
name: boot-test-${{ matrix.arch }}
path: boot-test-results.xml

View file

@ -341,4 +341,87 @@ hss-payload:
echo "Config: kernel/config/icicle_hss_payload.yaml (tear-de-silicio-compatible)"; \ echo "Config: kernel/config/icicle_hss_payload.yaml (tear-de-silicio-compatible)"; \
fi fi
.PHONY: all bin clean clean-all run-qemu run-qemu-firmware hss-payload run-linux run-linux-real run-personality # ---------------------------------------------------------------------------
# Test targets — build + boot smoke tests for CI
# ---------------------------------------------------------------------------
# ARCHS_BOOTTEST defines which architectures to test. Override on cmdline:
# make test ARCHS_BOOTTEST="armv7 aarch64"
ARCHS_BOOTTEST ?= armv7 aarch64 riscv
# QEMU binary and flags per architecture
QEMU_ARMV7 ?= qemu-system-arm
QEMU_ARMV7_FLAGS ?= -M virt -cpu cortex-a15 -m 512M -nographic
ARMV7_ELF = build/armv7/qemu-arm-virt/universalisos.elf
QEMU_AARCH64_TEST ?= qemu-system-aarch64
QEMU_AARCH64_FLAGS_TEST ?= -M virt,gic-version=3,virtualization=on -cpu cortex-a53 -m 512M -smp 4 -nographic
AARCH64_ELF = build/aarch64/qemu-aarch64-virt/universalisos.elf
# RISC-V uses the existing QEMU_RISCV settings
RISCV_ELF = build/riscv/polarfire/universalisos.elf
# Boot test timeout (seconds) — how long to wait for UART banner
BOOT_TIMEOUT ?= 15
# Banner string to grep for in UART output
BOOT_BANNER ?= UniversalisOS
# JUnit XML output for CI
TEST_RESULTS ?= $(BUILD_DIR)/test-results.xml
# Internal: run a single architecture boot test
# $(1) = arch name, $(2) = QEMU binary, $(3) = QEMU flags, $(4) = ELF path
define BOOT_TEST
@echo "=== Testing $(1): building... "
$(MAKE) ARCH=$(1) PLATFORM=qemu-$(1)-virt clean 2>/dev/null || true
$(MAKE) ARCH=$(1) PLATFORM=qemu-$(1)-virt
@echo "=== Testing $(1): booting in QEMU (timeout=$(BOOT_TIMEOUT)s)... "
@timeout $(BOOT_TIMEOUT) $(2) $(3) -kernel $(4) 2>&1 | tee /tmp/uos_boot_$(1).log ; \
exit_code=$$? ; \
if [ $$exit_code -eq 124 ]; then \
echo "=== $(1): TIMEOUT after $(BOOT_TIMEOUT)s ===" ; \
grep -q "$(BOOT_BANNER)" /tmp/uos_boot_$(1).log && \
echo "=== $(1): PASS (banner found before timeout) ===" || \
echo "=== $(1): FAIL (no banner found) ===" ; \
elif [ $$exit_code -eq 0 ]; then \
echo "=== $(1): PASS ===" ; \
else \
echo "=== $(1): FAIL (exit code $$exit_code) ===" ; \
fi
@grep -q "$(BOOT_BANNER)" /tmp/uos_boot_$(1).log 2>/dev/null
endef
test:
@echo "UniversalisOS Boot Test Suite"
@echo "============================="
@echo "Architectures: $(ARCHS_BOOTTEST)"
@echo "Timeout: $(BOOT_TIMEOUT)s per arch"
@echo ""
@failures=0 ; \
for arch in $(ARCHS_BOOTTEST); do \
case $$arch in \
armv7) $(call BOOT_TEST,armv7,$(QEMU_ARMV7),"$(QEMU_ARMV7_FLAGS)",$(ARMV7_ELF)) || failures=$$((failures+1)) ;; \
aarch64) $(call BOOT_TEST,aarch64,$(QEMU_AARCH64_TEST),"$(QEMU_AARCH64_FLAGS_TEST)",$(AARCH64_ELF)) || failures=$$((failures+1)) ;; \
riscv) $(call BOOT_TEST,riscv,$(QEMU_RISCV),"$(QEMU_RISCV_FLAGS) -bios none",$(RISCV_ELF)) || failures=$$((failures+1)) ;; \
*) echo "Unknown architecture: $$arch" ; failures=$$((failures+1)) ;; \
esac ; \
done ; \
echo "" ; \
echo "=============================" ; \
if [ $$failures -eq 0 ]; then \
echo "ALL TESTS PASSED" ; \
else \
echo "$$failures TEST(S) FAILED" ; \
exit 1 ; \
fi
test-armv7:
$(MAKE) test ARCHS_BOOTTEST="armv7"
test-aarch64:
$(MAKE) test ARCHS_BOOTTEST="aarch64"
test-riscv:
$(MAKE) test ARCHS_BOOTTEST="riscv"
.PHONY: all bin clean clean-all run-qemu run-qemu-firmware hss-payload run-linux run-linux-real run-personality test test-armv7 test-aarch64 test-riscv

View file

@ -0,0 +1,67 @@
# uos-boot-test
QEMU boot test orchestrator for UniversalisOS. Spawns QEMU for each architecture, monitors UART output for the boot banner, and reports pass/fail.
## Usage
```bash
# Test all architectures
python3 tools/uos-boot-test/uos-boot-test.py --arch all
# Test specific architecture
python3 tools/uos-boot-test/uos-boot-test.py --arch armv7
# Generate JUnit XML for CI
python3 tools/uos-boot-test/uos-boot-test.py --arch all --junit results.xml
# Custom timeout
python3 tools/uos-boot-test/uos-boot-test.py --arch armv7 --timeout 30
# Verbose output (show UART)
python3 tools/uos-boot-test/uos-boot-test.py --arch all --verbose
```
## How It Works
1. **Build**: Compiles the kernel for the target architecture (skipped with `--no-build`)
2. **Boot**: Launches QEMU with the compiled ELF
3. **Monitor**: Captures UART output for up to `--timeout` seconds
4. **Verify**: Checks for the "UniversalisOS" banner string in output
5. **Report**: Prints pass/fail and optionally generates JUnit XML
## Exit Codes
- `0` — All tests passed
- `1` — One or more tests failed
## CI Integration
The JUnit XML output integrates with GitHub Actions:
```yaml
- name: Run boot tests
run: python3 tools/uos-boot-test/uos-boot-test.py --arch all --junit test-results.xml
- name: Publish test results
uses: dorny/test-reporter@v1
if: always()
with:
name: Boot Tests
path: test-results.xml
reporter: java-junit
```
## Supported Architectures
| Architecture | QEMU Binary | Default Flags |
|---|---|---|
| `armv7` | `qemu-system-arm` | `-M virt -cpu cortex-a15 -m 512M -nographic` |
| `aarch64` | `qemu-system-aarch64` | `-M virt,gic-version=3,virtualization=on -cpu cortex-a53 -m 512M -smp 4 -nographic` |
| `riscv` | `qemu-system-riscv64` | `-machine microchip-icicle-kit -smp 5 -m 2G -nographic -bios none` |
## Limitations
- Only checks for boot banner (no functional test validation)
- RISC-V `-bios none` is BSP-only (secondaries held in reset)
- AArch64 requires `virtualization=on` (see AGENTS.md gotcha)
- Timeout may kill QEMU before banner appears on slow systems

View file

@ -0,0 +1,356 @@
#!/usr/bin/env python3
"""
uos-boot-test QEMU boot test orchestrator for UniversalisOS.
Spawns QEMU for each architecture, monitors UART output for the boot banner,
and reports pass/fail. Outputs JUnit XML for CI integration.
Usage:
python3 tools/uos-boot-test/uos-boot-test.py --arch armv7
python3 tools/uos-boot-test/uos-boot-test.py --arch all
python3 tools/uos-boot-test/uos-boot-test.py --arch armv7 --junit results.xml
"""
import argparse
import os
import subprocess
import sys
import time
import xml.etree.ElementTree as ET
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional
# ── Architecture Definitions ──────────────────────────────────────────
@dataclass
class ArchConfig:
"""Configuration for a target architecture."""
name: str
qemu_binary: str
qemu_flags: str
elf_path: str
platform: str
ARCHITECTURES = {
"armv7": ArchConfig(
name="armv7",
qemu_binary="qemu-system-arm",
qemu_flags="-M virt -cpu cortex-a15 -m 512M -nographic",
elf_path="kernel/build/armv7/qemu-arm-virt/universalisos.elf",
platform="qemu-arm-virt",
),
"aarch64": ArchConfig(
name="aarch64",
qemu_binary="qemu-system-aarch64",
qemu_flags="-M virt,gic-version=3,virtualization=on -cpu cortex-a53 -m 512M -smp 4 -nographic",
elf_path="kernel/build/aarch64/qemu-aarch64-virt/universalisos.elf",
platform="qemu-aarch64-virt",
),
"riscv": ArchConfig(
name="riscv",
qemu_binary="qemu-system-riscv64",
qemu_flags="-machine microchip-icicle-kit -smp 5 -m 2G -nographic -bios none",
elf_path="kernel/build/riscv/polarfire/universalisos.elf",
platform="polarfire",
),
}
# ── Test Result Types ─────────────────────────────────────────────────
@dataclass
class TestResult:
"""Result of a single boot test."""
arch: str
passed: bool
banner_found: bool
boot_time_seconds: float
timeout: bool
error_message: Optional[str] = None
uart_output: str = ""
# ── Core Functions ────────────────────────────────────────────────────
def find_repo_root() -> Path:
"""Find the repository root directory."""
current = Path(__file__).resolve().parent
while current != current.parent:
if (current / ".git").exists() or (current / "kernel").exists():
return current
current = current.parent
return Path.cwd()
def check_qemu_available(qemu_binary: str) -> bool:
"""Check if a QEMU binary is available."""
try:
result = subprocess.run(
[qemu_binary, "--version"],
capture_output=True,
timeout=5,
)
return result.returncode == 0
except (FileNotFoundError, subprocess.TimeoutExpired):
return False
def check_elf_exists(elf_path: Path) -> bool:
"""Check if the ELF binary exists."""
return elf_path.exists()
def build_kernel(arch: ArchConfig, repo_root: Path) -> bool:
"""Build the kernel for a specific architecture."""
kernel_dir = repo_root / "kernel"
print(f" Building {arch.name}...")
try:
result = subprocess.run(
["make", f"ARCH={arch.name}", f"PLATFORM={arch.platform}"],
cwd=kernel_dir,
capture_output=True,
timeout=300, # 5 minute build timeout
)
if result.returncode != 0:
print(f" Build failed for {arch.name}:")
print(f" {result.stderr.decode()[-500:]}")
return False
return True
except subprocess.TimeoutExpired:
print(f" Build timed out for {arch.name}")
return False
def run_boot_test(arch: ArchConfig, repo_root: Path, timeout: int = 15) -> TestResult:
"""Run a boot test for a single architecture."""
elf_path = repo_root / arch.elf_path
banner = "UniversalisOS"
# Check prerequisites
if not check_qemu_available(arch.qemu_binary):
return TestResult(
arch=arch.name,
passed=False,
banner_found=False,
boot_time_seconds=0,
timeout=False,
error_message=f"QEMU not found: {arch.qemu_binary}",
)
if not check_elf_exists(elf_path):
return TestResult(
arch=arch.name,
passed=False,
banner_found=False,
boot_time_seconds=0,
timeout=False,
error_message=f"ELF not found: {elf_path}",
)
# Build kernel
if not build_kernel(arch, repo_root):
return TestResult(
arch=arch.name,
passed=False,
banner_found=False,
boot_time_seconds=0,
timeout=False,
error_message="Build failed",
)
# Run QEMU and capture output
print(f" Booting {arch.name} in QEMU (timeout={timeout}s)...")
cmd = [arch.qemu_binary] + arch.qemu_flags.split() + ["-kernel", str(elf_path)]
start_time = time.time()
try:
result = subprocess.run(
cmd,
capture_output=True,
timeout=timeout,
cwd=repo_root,
)
boot_time = time.time() - start_time
output = result.stdout.decode(errors="replace") + result.stderr.decode(errors="replace")
# Check for timeout (process killed by timeout)
timed_out = result.returncode == -9 or result.returncode == 124
banner_found = banner in output
return TestResult(
arch=arch.name,
passed=banner_found,
banner_found=banner_found,
boot_time_seconds=boot_time,
timeout=timed_out,
uart_output=output[-2000:], # Last 2000 chars
)
except subprocess.TimeoutExpired:
boot_time = time.time() - start_time
return TestResult(
arch=arch.name,
passed=False,
banner_found=False,
boot_time_seconds=boot_time,
timeout=True,
error_message=f"QEMU process timed out after {timeout}s",
)
except Exception as e:
boot_time = time.time() - start_time
return TestResult(
arch=arch.name,
passed=False,
banner_found=False,
boot_time_seconds=boot_time,
timeout=False,
error_message=str(e),
)
# ── JUnit XML Output ─────────────────────────────────────────────────
def generate_junit_xml(results: list[TestResult], output_path: Path) -> None:
"""Generate JUnit XML test report."""
testsuite = ET.Element("testsuite")
testsuite.set("name", "UniversalisOS Boot Tests")
testsuite.set("tests", str(len(results)))
testsuite.set("failures", str(sum(1 for r in results if not r.passed)))
testsuite.set("errors", str(sum(1 for r in results if r.error_message and not r.timeout)))
testsuite.set("time", f"{sum(r.boot_time_seconds for r in results):.2f}")
for result in results:
testcase = ET.SubElement(testsuite, "testcase")
testcase.set("name", f"boot-{result.arch}")
testcase.set("classname", "uos-boot-test")
testcase.set("time", f"{result.boot_time_seconds:.2f}")
if not result.passed:
if result.timeout:
failure = ET.SubElement(testcase, "failure")
failure.set("message", f"Boot test timed out after {result.boot_time_seconds:.1f}s")
failure.set("type", "timeout")
failure.text = f"Banner 'UniversalisOS' not found in UART output within timeout.\n\nUART output (last 500 chars):\n{result.uart_output[-500:]}"
else:
failure = ET.SubElement(testcase, "failure")
failure.set("message", result.error_message or "Boot test failed")
failure.set("type", "assertion")
failure.text = f"Banner 'UniversalisOS' not found in UART output.\n\nUART output (last 500 chars):\n{result.uart_output[-500:]}"
tree = ET.ElementTree(testsuite)
ET.indent(tree, space=" ")
tree.write(output_path, encoding="unicode", xml_declaration=True)
# ── Console Output ────────────────────────────────────────────────────
def print_results(results: list[TestResult]) -> None:
"""Print test results to console."""
print("\n" + "=" * 60)
print("UniversalisOS Boot Test Results")
print("=" * 60)
for result in results:
status = "PASS" if result.passed else "FAIL"
icon = "" if result.passed else ""
time_str = f"{result.boot_time_seconds:.1f}s"
print(f" {icon} {result.arch:<10} {status:<6} ({time_str})")
if result.error_message:
print(f" Error: {result.error_message}")
if result.timeout:
print(f" Note: QEMU was killed after timeout; banner may appear later on real hardware")
passed = sum(1 for r in results if r.passed)
total = len(results)
print("=" * 60)
print(f" {passed}/{total} tests passed")
print("=" * 60)
# ── Main ──────────────────────────────────────────────────────────────
def main():
parser = argparse.ArgumentParser(
description="UniversalisOS boot test orchestrator",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
%(prog)s --arch armv7 # Test ARMv7 only
%(prog)s --arch all # Test all architectures
%(prog)s --arch all --junit results.xml # Generate JUnit XML
%(prog)s --arch armv7 --timeout 30 # Custom timeout
""",
)
parser.add_argument(
"--arch",
choices=["armv7", "aarch64", "riscv", "all"],
default="all",
help="Architecture to test (default: all)",
)
parser.add_argument(
"--timeout",
type=int,
default=15,
help="Boot timeout in seconds (default: 15)",
)
parser.add_argument(
"--junit",
type=str,
default=None,
help="Output JUnit XML to this file",
)
parser.add_argument(
"--no-build",
action="store_true",
help="Skip kernel build (use existing ELF)",
)
parser.add_argument(
"--verbose",
action="store_true",
help="Show full UART output",
)
args = parser.parse_args()
repo_root = find_repo_root()
# Determine architectures to test
if args.arch == "all":
archs = list(ARCHITECTURES.values())
else:
archs = [ARCHITECTURES[args.arch]]
print(f"UniversalisOS Boot Test Orchestrator")
print(f"Repository: {repo_root}")
print(f"Architectures: {[a.name for a in archs]}")
print(f"Timeout: {args.timeout}s per architecture")
print()
# Run tests
results = []
for arch in archs:
result = run_boot_test(arch, repo_root, args.timeout)
results.append(result)
if args.verbose and result.uart_output:
print(f"\n--- UART output for {arch.name} ---")
print(result.uart_output[-1000:])
print(f"--- end {arch.name} ---\n")
# Print results
print_results(results)
# Generate JUnit XML if requested
if args.junit:
junit_path = Path(args.junit)
junit_path.parent.mkdir(parents=True, exist_ok=True)
generate_junit_xml(results, junit_path)
print(f"\nJUnit XML written to: {junit_path}")
# Exit with failure if any test failed
if not all(r.passed for r in results):
sys.exit(1)
if __name__ == "__main__":
main()