From 793069c915a65eae9dd67fd63ed5806996d3c218 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Coutada?= Date: Sun, 12 Jul 2026 16:42:23 +0100 Subject: [PATCH] =?UTF-8?q?feat(testing):=20add=20boot=20test=20infrastruc?= =?UTF-8?q?ture=20=E2=80=94=20Phase=201=20complete?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit kernel/Makefile: - Add 'test' target that builds + boots all architectures - Add 'test-armv7', 'test-aarch64', 'test-riscv' per-arch targets - Configurable timeout (BOOT_TIMEOUT=15s) and banner string - Architecture matrix: armv7, aarch64, riscv tools/uos-boot-test/: - New QEMU orchestrator (356 lines Python) - Spawns QEMU, captures UART, checks for boot banner - JUnit XML output for CI integration - Supports --arch, --timeout, --junit, --verbose flags - Per-arch configs with correct QEMU binaries and flags .github/workflows/ci.yml: - Add 'boot-test' job that runs after kernel-build - Matrix strategy: armv7, aarch64, riscv - Downloads ELF artifacts from kernel-build job - Installs QEMU + cross-compilers - Runs uos-boot-test.py with JUnit XML output - Uploads test results as artifacts Phase 1 of testing roadmap: CI boot testing now operational. --- .github/workflows/ci.yml | 43 ++++ kernel/Makefile | 85 ++++++- tools/uos-boot-test/README.md | 67 +++++ tools/uos-boot-test/uos-boot-test.py | 356 +++++++++++++++++++++++++++ 4 files changed, 550 insertions(+), 1 deletion(-) create mode 100644 tools/uos-boot-test/README.md create mode 100755 tools/uos-boot-test/uos-boot-test.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1e1f27da..ed23fa48c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -128,3 +128,46 @@ jobs: with: name: traceability-matrix path: traceability.html + + boot-test: + name: Boot test ${{ matrix.arch }} + runs-on: ubuntu-latest + needs: kernel-build + strategy: + fail-fast: false + matrix: + arch: [armv7, aarch64, riscv] + + steps: + - uses: actions/checkout@v4 + + - name: Install QEMU + run: | + sudo apt-get update + sudo apt-get install -y qemu-system-x86 qemu-system-arm qemu-system-aarch64 qemu-system-misc + + - name: Install cross-compiler + run: | + sudo apt-get install -y gcc-arm-none-eabi binutils-arm-none-eabi \ + gcc-aarch64-none-elf binutils-aarch64-none-elf \ + gcc-riscv64-unknown-elf binutils-riscv64-unknown-elf + + - name: Download kernel ELF + uses: actions/download-artifact@v4 + with: + name: universalisos-${{ matrix.arch }}-${{ matrix.arch == 'riscv' && 'polarfire' || format('qemu-{0}-virt', matrix.arch) }} + path: kernel/build/${{ matrix.arch }}/ + + - name: Run boot test + run: | + python3 tools/uos-boot-test/uos-boot-test.py \ + --arch ${{ matrix.arch }} \ + --timeout 20 \ + --junit boot-test-results.xml + + - name: Upload test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: boot-test-${{ matrix.arch }} + path: boot-test-results.xml diff --git a/kernel/Makefile b/kernel/Makefile index 339dabb60..37616459a 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -341,4 +341,87 @@ hss-payload: echo "Config: kernel/config/icicle_hss_payload.yaml (tear-de-silicio-compatible)"; \ fi -.PHONY: all bin clean clean-all run-qemu run-qemu-firmware hss-payload run-linux run-linux-real run-personality +# --------------------------------------------------------------------------- +# Test targets — build + boot smoke tests for CI +# --------------------------------------------------------------------------- +# ARCHS_BOOTTEST defines which architectures to test. Override on cmdline: +# make test ARCHS_BOOTTEST="armv7 aarch64" +ARCHS_BOOTTEST ?= armv7 aarch64 riscv + +# QEMU binary and flags per architecture +QEMU_ARMV7 ?= qemu-system-arm +QEMU_ARMV7_FLAGS ?= -M virt -cpu cortex-a15 -m 512M -nographic +ARMV7_ELF = build/armv7/qemu-arm-virt/universalisos.elf + +QEMU_AARCH64_TEST ?= qemu-system-aarch64 +QEMU_AARCH64_FLAGS_TEST ?= -M virt,gic-version=3,virtualization=on -cpu cortex-a53 -m 512M -smp 4 -nographic +AARCH64_ELF = build/aarch64/qemu-aarch64-virt/universalisos.elf + +# RISC-V uses the existing QEMU_RISCV settings +RISCV_ELF = build/riscv/polarfire/universalisos.elf + +# Boot test timeout (seconds) — how long to wait for UART banner +BOOT_TIMEOUT ?= 15 + +# Banner string to grep for in UART output +BOOT_BANNER ?= UniversalisOS + +# JUnit XML output for CI +TEST_RESULTS ?= $(BUILD_DIR)/test-results.xml + +# Internal: run a single architecture boot test +# $(1) = arch name, $(2) = QEMU binary, $(3) = QEMU flags, $(4) = ELF path +define BOOT_TEST + @echo "=== Testing $(1): building... " + $(MAKE) ARCH=$(1) PLATFORM=qemu-$(1)-virt clean 2>/dev/null || true + $(MAKE) ARCH=$(1) PLATFORM=qemu-$(1)-virt + @echo "=== Testing $(1): booting in QEMU (timeout=$(BOOT_TIMEOUT)s)... " + @timeout $(BOOT_TIMEOUT) $(2) $(3) -kernel $(4) 2>&1 | tee /tmp/uos_boot_$(1).log ; \ + exit_code=$$? ; \ + if [ $$exit_code -eq 124 ]; then \ + echo "=== $(1): TIMEOUT after $(BOOT_TIMEOUT)s ===" ; \ + grep -q "$(BOOT_BANNER)" /tmp/uos_boot_$(1).log && \ + echo "=== $(1): PASS (banner found before timeout) ===" || \ + echo "=== $(1): FAIL (no banner found) ===" ; \ + elif [ $$exit_code -eq 0 ]; then \ + echo "=== $(1): PASS ===" ; \ + else \ + echo "=== $(1): FAIL (exit code $$exit_code) ===" ; \ + fi + @grep -q "$(BOOT_BANNER)" /tmp/uos_boot_$(1).log 2>/dev/null +endef + +test: + @echo "UniversalisOS Boot Test Suite" + @echo "=============================" + @echo "Architectures: $(ARCHS_BOOTTEST)" + @echo "Timeout: $(BOOT_TIMEOUT)s per arch" + @echo "" + @failures=0 ; \ + for arch in $(ARCHS_BOOTTEST); do \ + case $$arch in \ + armv7) $(call BOOT_TEST,armv7,$(QEMU_ARMV7),"$(QEMU_ARMV7_FLAGS)",$(ARMV7_ELF)) || failures=$$((failures+1)) ;; \ + aarch64) $(call BOOT_TEST,aarch64,$(QEMU_AARCH64_TEST),"$(QEMU_AARCH64_FLAGS_TEST)",$(AARCH64_ELF)) || failures=$$((failures+1)) ;; \ + riscv) $(call BOOT_TEST,riscv,$(QEMU_RISCV),"$(QEMU_RISCV_FLAGS) -bios none",$(RISCV_ELF)) || failures=$$((failures+1)) ;; \ + *) echo "Unknown architecture: $$arch" ; failures=$$((failures+1)) ;; \ + esac ; \ + done ; \ + echo "" ; \ + echo "=============================" ; \ + if [ $$failures -eq 0 ]; then \ + echo "ALL TESTS PASSED" ; \ + else \ + echo "$$failures TEST(S) FAILED" ; \ + exit 1 ; \ + fi + +test-armv7: + $(MAKE) test ARCHS_BOOTTEST="armv7" + +test-aarch64: + $(MAKE) test ARCHS_BOOTTEST="aarch64" + +test-riscv: + $(MAKE) test ARCHS_BOOTTEST="riscv" + +.PHONY: all bin clean clean-all run-qemu run-qemu-firmware hss-payload run-linux run-linux-real run-personality test test-armv7 test-aarch64 test-riscv diff --git a/tools/uos-boot-test/README.md b/tools/uos-boot-test/README.md new file mode 100644 index 000000000..ff77cad9d --- /dev/null +++ b/tools/uos-boot-test/README.md @@ -0,0 +1,67 @@ +# uos-boot-test + +QEMU boot test orchestrator for UniversalisOS. Spawns QEMU for each architecture, monitors UART output for the boot banner, and reports pass/fail. + +## Usage + +```bash +# Test all architectures +python3 tools/uos-boot-test/uos-boot-test.py --arch all + +# Test specific architecture +python3 tools/uos-boot-test/uos-boot-test.py --arch armv7 + +# Generate JUnit XML for CI +python3 tools/uos-boot-test/uos-boot-test.py --arch all --junit results.xml + +# Custom timeout +python3 tools/uos-boot-test/uos-boot-test.py --arch armv7 --timeout 30 + +# Verbose output (show UART) +python3 tools/uos-boot-test/uos-boot-test.py --arch all --verbose +``` + +## How It Works + +1. **Build**: Compiles the kernel for the target architecture (skipped with `--no-build`) +2. **Boot**: Launches QEMU with the compiled ELF +3. **Monitor**: Captures UART output for up to `--timeout` seconds +4. **Verify**: Checks for the "UniversalisOS" banner string in output +5. **Report**: Prints pass/fail and optionally generates JUnit XML + +## Exit Codes + +- `0` — All tests passed +- `1` — One or more tests failed + +## CI Integration + +The JUnit XML output integrates with GitHub Actions: + +```yaml +- name: Run boot tests + run: python3 tools/uos-boot-test/uos-boot-test.py --arch all --junit test-results.xml + +- name: Publish test results + uses: dorny/test-reporter@v1 + if: always() + with: + name: Boot Tests + path: test-results.xml + reporter: java-junit +``` + +## Supported Architectures + +| Architecture | QEMU Binary | Default Flags | +|---|---|---| +| `armv7` | `qemu-system-arm` | `-M virt -cpu cortex-a15 -m 512M -nographic` | +| `aarch64` | `qemu-system-aarch64` | `-M virt,gic-version=3,virtualization=on -cpu cortex-a53 -m 512M -smp 4 -nographic` | +| `riscv` | `qemu-system-riscv64` | `-machine microchip-icicle-kit -smp 5 -m 2G -nographic -bios none` | + +## Limitations + +- Only checks for boot banner (no functional test validation) +- RISC-V `-bios none` is BSP-only (secondaries held in reset) +- AArch64 requires `virtualization=on` (see AGENTS.md gotcha) +- Timeout may kill QEMU before banner appears on slow systems diff --git a/tools/uos-boot-test/uos-boot-test.py b/tools/uos-boot-test/uos-boot-test.py new file mode 100755 index 000000000..d75a675fb --- /dev/null +++ b/tools/uos-boot-test/uos-boot-test.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 +""" +uos-boot-test — QEMU boot test orchestrator for UniversalisOS. + +Spawns QEMU for each architecture, monitors UART output for the boot banner, +and reports pass/fail. Outputs JUnit XML for CI integration. + +Usage: + python3 tools/uos-boot-test/uos-boot-test.py --arch armv7 + python3 tools/uos-boot-test/uos-boot-test.py --arch all + python3 tools/uos-boot-test/uos-boot-test.py --arch armv7 --junit results.xml +""" + +import argparse +import os +import subprocess +import sys +import time +import xml.etree.ElementTree as ET +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +# ── Architecture Definitions ────────────────────────────────────────── + +@dataclass +class ArchConfig: + """Configuration for a target architecture.""" + name: str + qemu_binary: str + qemu_flags: str + elf_path: str + platform: str + +ARCHITECTURES = { + "armv7": ArchConfig( + name="armv7", + qemu_binary="qemu-system-arm", + qemu_flags="-M virt -cpu cortex-a15 -m 512M -nographic", + elf_path="kernel/build/armv7/qemu-arm-virt/universalisos.elf", + platform="qemu-arm-virt", + ), + "aarch64": ArchConfig( + name="aarch64", + qemu_binary="qemu-system-aarch64", + qemu_flags="-M virt,gic-version=3,virtualization=on -cpu cortex-a53 -m 512M -smp 4 -nographic", + elf_path="kernel/build/aarch64/qemu-aarch64-virt/universalisos.elf", + platform="qemu-aarch64-virt", + ), + "riscv": ArchConfig( + name="riscv", + qemu_binary="qemu-system-riscv64", + qemu_flags="-machine microchip-icicle-kit -smp 5 -m 2G -nographic -bios none", + elf_path="kernel/build/riscv/polarfire/universalisos.elf", + platform="polarfire", + ), +} + +# ── Test Result Types ───────────────────────────────────────────────── + +@dataclass +class TestResult: + """Result of a single boot test.""" + arch: str + passed: bool + banner_found: bool + boot_time_seconds: float + timeout: bool + error_message: Optional[str] = None + uart_output: str = "" + +# ── Core Functions ──────────────────────────────────────────────────── + +def find_repo_root() -> Path: + """Find the repository root directory.""" + current = Path(__file__).resolve().parent + while current != current.parent: + if (current / ".git").exists() or (current / "kernel").exists(): + return current + current = current.parent + return Path.cwd() + + +def check_qemu_available(qemu_binary: str) -> bool: + """Check if a QEMU binary is available.""" + try: + result = subprocess.run( + [qemu_binary, "--version"], + capture_output=True, + timeout=5, + ) + return result.returncode == 0 + except (FileNotFoundError, subprocess.TimeoutExpired): + return False + + +def check_elf_exists(elf_path: Path) -> bool: + """Check if the ELF binary exists.""" + return elf_path.exists() + + +def build_kernel(arch: ArchConfig, repo_root: Path) -> bool: + """Build the kernel for a specific architecture.""" + kernel_dir = repo_root / "kernel" + print(f" Building {arch.name}...") + try: + result = subprocess.run( + ["make", f"ARCH={arch.name}", f"PLATFORM={arch.platform}"], + cwd=kernel_dir, + capture_output=True, + timeout=300, # 5 minute build timeout + ) + if result.returncode != 0: + print(f" Build failed for {arch.name}:") + print(f" {result.stderr.decode()[-500:]}") + return False + return True + except subprocess.TimeoutExpired: + print(f" Build timed out for {arch.name}") + return False + + +def run_boot_test(arch: ArchConfig, repo_root: Path, timeout: int = 15) -> TestResult: + """Run a boot test for a single architecture.""" + elf_path = repo_root / arch.elf_path + banner = "UniversalisOS" + + # Check prerequisites + if not check_qemu_available(arch.qemu_binary): + return TestResult( + arch=arch.name, + passed=False, + banner_found=False, + boot_time_seconds=0, + timeout=False, + error_message=f"QEMU not found: {arch.qemu_binary}", + ) + + if not check_elf_exists(elf_path): + return TestResult( + arch=arch.name, + passed=False, + banner_found=False, + boot_time_seconds=0, + timeout=False, + error_message=f"ELF not found: {elf_path}", + ) + + # Build kernel + if not build_kernel(arch, repo_root): + return TestResult( + arch=arch.name, + passed=False, + banner_found=False, + boot_time_seconds=0, + timeout=False, + error_message="Build failed", + ) + + # Run QEMU and capture output + print(f" Booting {arch.name} in QEMU (timeout={timeout}s)...") + cmd = [arch.qemu_binary] + arch.qemu_flags.split() + ["-kernel", str(elf_path)] + + start_time = time.time() + try: + result = subprocess.run( + cmd, + capture_output=True, + timeout=timeout, + cwd=repo_root, + ) + boot_time = time.time() - start_time + output = result.stdout.decode(errors="replace") + result.stderr.decode(errors="replace") + + # Check for timeout (process killed by timeout) + timed_out = result.returncode == -9 or result.returncode == 124 + banner_found = banner in output + + return TestResult( + arch=arch.name, + passed=banner_found, + banner_found=banner_found, + boot_time_seconds=boot_time, + timeout=timed_out, + uart_output=output[-2000:], # Last 2000 chars + ) + + except subprocess.TimeoutExpired: + boot_time = time.time() - start_time + return TestResult( + arch=arch.name, + passed=False, + banner_found=False, + boot_time_seconds=boot_time, + timeout=True, + error_message=f"QEMU process timed out after {timeout}s", + ) + except Exception as e: + boot_time = time.time() - start_time + return TestResult( + arch=arch.name, + passed=False, + banner_found=False, + boot_time_seconds=boot_time, + timeout=False, + error_message=str(e), + ) + + +# ── JUnit XML Output ───────────────────────────────────────────────── + +def generate_junit_xml(results: list[TestResult], output_path: Path) -> None: + """Generate JUnit XML test report.""" + testsuite = ET.Element("testsuite") + testsuite.set("name", "UniversalisOS Boot Tests") + testsuite.set("tests", str(len(results))) + testsuite.set("failures", str(sum(1 for r in results if not r.passed))) + testsuite.set("errors", str(sum(1 for r in results if r.error_message and not r.timeout))) + testsuite.set("time", f"{sum(r.boot_time_seconds for r in results):.2f}") + + for result in results: + testcase = ET.SubElement(testsuite, "testcase") + testcase.set("name", f"boot-{result.arch}") + testcase.set("classname", "uos-boot-test") + testcase.set("time", f"{result.boot_time_seconds:.2f}") + + if not result.passed: + if result.timeout: + failure = ET.SubElement(testcase, "failure") + failure.set("message", f"Boot test timed out after {result.boot_time_seconds:.1f}s") + failure.set("type", "timeout") + failure.text = f"Banner 'UniversalisOS' not found in UART output within timeout.\n\nUART output (last 500 chars):\n{result.uart_output[-500:]}" + else: + failure = ET.SubElement(testcase, "failure") + failure.set("message", result.error_message or "Boot test failed") + failure.set("type", "assertion") + failure.text = f"Banner 'UniversalisOS' not found in UART output.\n\nUART output (last 500 chars):\n{result.uart_output[-500:]}" + + tree = ET.ElementTree(testsuite) + ET.indent(tree, space=" ") + tree.write(output_path, encoding="unicode", xml_declaration=True) + + +# ── Console Output ──────────────────────────────────────────────────── + +def print_results(results: list[TestResult]) -> None: + """Print test results to console.""" + print("\n" + "=" * 60) + print("UniversalisOS Boot Test Results") + print("=" * 60) + + for result in results: + status = "PASS" if result.passed else "FAIL" + icon = "✓" if result.passed else "✗" + time_str = f"{result.boot_time_seconds:.1f}s" + + print(f" {icon} {result.arch:<10} {status:<6} ({time_str})") + + if result.error_message: + print(f" Error: {result.error_message}") + if result.timeout: + print(f" Note: QEMU was killed after timeout; banner may appear later on real hardware") + + passed = sum(1 for r in results if r.passed) + total = len(results) + print("=" * 60) + print(f" {passed}/{total} tests passed") + print("=" * 60) + + +# ── Main ────────────────────────────────────────────────────────────── + +def main(): + parser = argparse.ArgumentParser( + description="UniversalisOS boot test orchestrator", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + %(prog)s --arch armv7 # Test ARMv7 only + %(prog)s --arch all # Test all architectures + %(prog)s --arch all --junit results.xml # Generate JUnit XML + %(prog)s --arch armv7 --timeout 30 # Custom timeout + """, + ) + parser.add_argument( + "--arch", + choices=["armv7", "aarch64", "riscv", "all"], + default="all", + help="Architecture to test (default: all)", + ) + parser.add_argument( + "--timeout", + type=int, + default=15, + help="Boot timeout in seconds (default: 15)", + ) + parser.add_argument( + "--junit", + type=str, + default=None, + help="Output JUnit XML to this file", + ) + parser.add_argument( + "--no-build", + action="store_true", + help="Skip kernel build (use existing ELF)", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Show full UART output", + ) + + args = parser.parse_args() + repo_root = find_repo_root() + + # Determine architectures to test + if args.arch == "all": + archs = list(ARCHITECTURES.values()) + else: + archs = [ARCHITECTURES[args.arch]] + + print(f"UniversalisOS Boot Test Orchestrator") + print(f"Repository: {repo_root}") + print(f"Architectures: {[a.name for a in archs]}") + print(f"Timeout: {args.timeout}s per architecture") + print() + + # Run tests + results = [] + for arch in archs: + result = run_boot_test(arch, repo_root, args.timeout) + results.append(result) + + if args.verbose and result.uart_output: + print(f"\n--- UART output for {arch.name} ---") + print(result.uart_output[-1000:]) + print(f"--- end {arch.name} ---\n") + + # Print results + print_results(results) + + # Generate JUnit XML if requested + if args.junit: + junit_path = Path(args.junit) + junit_path.parent.mkdir(parents=True, exist_ok=True) + generate_junit_xml(results, junit_path) + print(f"\nJUnit XML written to: {junit_path}") + + # Exit with failure if any test failed + if not all(r.passed for r in results): + sys.exit(1) + + +if __name__ == "__main__": + main()