From 3a6c31a51e52b582a1e3fca46489d56cdb048b98 Mon Sep 17 00:00:00 2001 From: Juliusz Sosinowicz Date: Thu, 11 Jun 2026 17:13:58 +0000 Subject: [PATCH 1/9] CI: pool the per-config runner matrices into parallel make-check jobs Replace the one-runner-per-configuration matrices across the make-check workflow family with a generic pooled runner, .github/scripts/parallel-make-check.py. Each workflow keeps its configuration list as JSON next to the invocation; one runner (or a small fixed set of shards, balanced by measured per-config minutes) builds every config in its own out-of-tree (VPATH) build directory off a single checkout/autogen, on a pool of one-per-CPU worker threads, longest first. Concurrent checks are isolated with bubblewrap network namespaces, compilations are cached with ccache, the first failure aborts the rest (fail-fast, with --no-fail-fast to run everything), and per-config timings plus pool efficiency land in the step summary. Failure logs upload as artifacts. smoke-test.yml is likewise reworked into a single pooled job that runs its nine configs on one runner. Converted workflows (runner jobs per full pass): os-check.yml 101 -> 8 (92 Ubuntu configs -> 4 shards; the macOS matrix, the user-settings jobs and the standalone macos-apple-native-cert-validation.yml fold into one macOS runner; Windows unchanged) pq-all.yml 21 -> 2 shards disable-pk-algs.yml 15 -> 1 wolfCrypt-Wconversion.yml 11 -> 1 trackmemory.yml 7 -> 1 cryptocb-only.yml 8 -> 1 (incl. the two new SHA512 entries) multi-compiler.yml 6 -> 1 smallStackSize.yml 6 -> 1 multi-arch.yml 6 -> 1 async.yml 5 -> 1 psk.yml 5 -> 1 no-malloc.yml 3 -> 1 wolfsm.yml 3 -> 1 opensslcoexist.yml 2 -> 1 Measured against current upstream passing runs (job execution time, queue excluded): ~200 runner jobs / ~374 runner-minutes per full pass become 23 jobs / ~168 runner-minutes, with more coverage than before. multi-arch's old matrix combined an "include" list of four architectures with an "opts" axis; GitHub's include-merge rules made each arch entry overwrite the previous one, so only the armel combinations actually ran. The pooled list restores the intended aarch64/armhf/riscv64 coverage (23 combinations; riscv64 x sp-math is omitted as invalid - configure rejects sp-math without SP, and --enable-riscv-asm, unlike --enable-sp-asm, does not bring SP in). Out-of-tree build fixes this depends on: - Makefile.am: symlink the read-only test data (certs/, tests/ config files, sniffer captures and helpers, examples/crypto_policies, input, quit) into the build tree via a BUILT_SOURCES stamp, removed again in distclean-local. ChangeToWolfRoot() and the script tests resolve everything relative to the working directory, so out-of-tree make check and make distcheck now pass. - scripts/multi-msg-record.py: locate the client binary from the build tree working directory rather than the script's source directory. - configure.ac + wolfssl/include.am: run support/gen-debug-trace-error-codes.sh from $srcdir; it reads the error-code headers from the source tree and generates into the build tree. - tests/swdev: a WOLFBUILD variable points the sub-make at the build tree for the configure-generated headers (wolfssl/options.h, wolfssl/version.h); the in-tree-only guards are dropped. Portions of PR #10649 are incorporated: the cross-platform ccache-setup composite action, repository_owner gates on check-headers and check-source-text, the docs-only paths-ignore on os-check, and the libspdm timeout bumps. --- .github/actions/ccache-setup/action.yml | 34 +- .github/actions/wait-for-smoke/action.yml | 2 +- .github/scripts/parallel-make-check.py | 432 +++++++++++ .github/workflows/async.yml | 98 ++- .github/workflows/check-headers.yml | 2 + .github/workflows/check-source-text.yml | 2 + .github/workflows/cryptocb-only.yml | 296 +++++--- .github/workflows/disable-pk-algs.yml | 155 +++- .github/workflows/libspdm.yml | 10 +- .../macos-apple-native-cert-validation.yml | 27 - .github/workflows/multi-arch.yml | 296 ++++++-- .github/workflows/multi-compiler.yml | 116 ++- .github/workflows/no-malloc.yml | 84 ++- .github/workflows/opensslcoexist.yml | 90 ++- .github/workflows/os-check.yml | 701 +++++++++++------- .github/workflows/pq-all.yml | 258 +++++-- .github/workflows/psk.yml | 116 ++- .github/workflows/smallStackSize.yml | 140 +++- .github/workflows/smoke-test.yml | 135 ++-- .github/workflows/trackmemory.yml | 119 ++- .github/workflows/wolfCrypt-Wconversion.yml | 155 +++- .github/workflows/wolfsm.yml | 98 ++- .gitignore | 3 + Makefile.am | 68 ++ configure.ac | 5 +- scripts/multi-msg-record.py | 15 +- support/gen-debug-trace-error-codes.sh | 11 +- tests/swdev/Makefile | 7 +- tests/swdev/README.md | 9 +- wolfcrypt/test/include.am | 5 +- wolfssl/include.am | 2 +- 31 files changed, 2641 insertions(+), 850 deletions(-) create mode 100755 .github/scripts/parallel-make-check.py delete mode 100644 .github/workflows/macos-apple-native-cert-validation.yml diff --git a/.github/actions/ccache-setup/action.yml b/.github/actions/ccache-setup/action.yml index 206bfb66d5..73b8d5214f 100644 --- a/.github/actions/ccache-setup/action.yml +++ b/.github/actions/ccache-setup/action.yml @@ -1,9 +1,10 @@ name: 'Set up ccache' description: > - Install ccache (on Ubuntu), restore the ccache directory from a previous - run, and prepend the ccache compiler-symlink dir to PATH. Subsequent - gcc/cc/g++/c++ invocations are transparently intercepted by ccache, so - no other workflow step needs to change. macOS is not supported yet. + Install ccache (Ubuntu via apt, macOS via brew), restore the ccache + directory from a previous run, and prepend the ccache compiler-symlink + dir to PATH. Subsequent gcc/cc/g++/c++/clang invocations are + transparently intercepted by ccache, so no other workflow step needs to + change. inputs: workflow-id: @@ -24,15 +25,20 @@ inputs: runs: using: 'composite' steps: - - name: Install ccache (Ubuntu) + - name: Install ccache shell: bash run: | if command -v ccache >/dev/null 2>&1; then echo "ccache already installed: $(ccache --version | head -1)" - else + elif [ "${{ runner.os }}" = "Linux" ]; then sudo apt-get update -q sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ --no-install-recommends ccache + elif [ "${{ runner.os }}" = "macOS" ]; then + brew install ccache + else + echo "::error::ccache install not supported on ${{ runner.os }}" + exit 1 fi - name: Restore + save ccache @@ -56,10 +62,18 @@ runs: ccache --set-config=base_dir="$GITHUB_WORKSPACE" ccache --set-config=hash_dir=false ccache -z # zero stats so the post-build summary is per-job - # /usr/lib/ccache contains gcc, g++, cc, c++ symlinks that resolve - # to ccache. Prepending to PATH makes the build transparently use - # ccache without changing any configure/make invocation. - echo "/usr/lib/ccache" >> "$GITHUB_PATH" + # The ccache compiler-symlink dir (gcc, g++, cc, c++, clang, ... + # all resolving to ccache) differs by platform. Prepending it to + # PATH makes the build transparently use ccache without changing + # any configure/make invocation. On macOS the symlinks live under + # the Homebrew libexec dir, whose prefix is /opt/homebrew on arm64 + # and /usr/local on Intel - resolve it via `brew --prefix`. + if [ "${{ runner.os }}" = "macOS" ]; then + CCACHE_LIBEXEC="$(brew --prefix)/opt/ccache/libexec" + else + CCACHE_LIBEXEC="/usr/lib/ccache" + fi + echo "$CCACHE_LIBEXEC" >> "$GITHUB_PATH" echo "CCACHE_DIR=$HOME/.ccache" >> "$GITHUB_ENV" - name: Show ccache stats (initial) diff --git a/.github/actions/wait-for-smoke/action.yml b/.github/actions/wait-for-smoke/action.yml index 13c15b7604..0a6793b24d 100644 --- a/.github/actions/wait-for-smoke/action.yml +++ b/.github/actions/wait-for-smoke/action.yml @@ -17,7 +17,7 @@ inputs: timeout-seconds: description: 'Maximum time to wait for smoke to complete' required: false - default: '1800' + default: '3600' poll-seconds: description: 'Polling interval' required: false diff --git a/.github/scripts/parallel-make-check.py b/.github/scripts/parallel-make-check.py new file mode 100755 index 0000000000..4f4cf2ce82 --- /dev/null +++ b/.github/scripts/parallel-make-check.py @@ -0,0 +1,432 @@ +#!/usr/bin/env python3 +# Build and "make check" a set of configurations, each in its own out-of-tree +# (VPATH) build directory, on a pool of worker threads (default: one per +# CPU); each thread takes the next pending config as soon as it is free. +# The final summary reports how efficiently the pool used the machine +# (thread occupancy and CPU utilization). +# +# The configurations come from a JSON file ("-" for stdin): a list of +# objects, one per configuration. Recognized keys, all optional except +# "name" (unknown keys are an error, so typos do not pass silently): +# +# name unique identifier; the config builds in build-/ +# configure list of extra ./configure arguments +# cc compiler passed to configure as CC=, overriding --cc +# ("" leaves CC entirely to configure / the environment) +# cflags CFLAGS for make, overriding --cflags +# ldflags LDFLAGS for make, overriding --ldflags +# minutes expected duration, from the Minutes column of a previous +# run's summary (default 1.0). Schedule weight only - configs +# run longest-first and --shard balances shards by it; a stale +# value just packs the schedule a little worse. +# user_settings header staged as /user_settings.h before +# configure (path relative to the source root); pair it with +# --enable-usersettings in "configure" +# check false skips the make-check phase entirely (default true) +# prepare list of argv lists run in the build dir before configure +# run list of argv lists run in the build dir after the build and +# checks, e.g. [["wolfcrypt/test/testwolfcrypt"]] +# comment ignored; JSON has no comment syntax, so notes go here +# +# For example: +# +# [ +# {"name": "default"}, +# {"name": "all-asan", "configure": ["--enable-all"], +# "cflags": "-fsanitize=address", "ldflags": "-fsanitize=address"} +# ] +# +# Driven by CI workflows, which keep their config lists next to the +# invocation (see .github/workflows/smoke-test.yml), but also runnable +# locally - copy the JSON block out of the workflow into a file: +# +# .github/scripts/parallel-make-check.py configs.json # all configs +# .github/scripts/parallel-make-check.py configs.json default all-asan +# .github/scripts/parallel-make-check.py --list configs.json +# +# Concurrent "make check" runs are safe because the test scripts re-exec +# themselves under "bwrap --unshare-net" when bubblewrap is installed (one +# network namespace each) and the remaining test outputs land in the build +# directory; see --private-dir for the exception. +# +# The first failing config aborts the others (pending configs are skipped, +# in-flight ones are killed) so CI fails fast; pass --no-fail-fast to run +# everything and report every failure. + +import argparse +import json +import os +import shutil +import signal +import subprocess +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from pathlib import Path + +# cflags/ldflags are applied at make time only (never to ./configure) so +# autoconf feature detection is not poisoned by benign warnings in +# conftest probes. They are omitted entirely when empty so a plain config +# keeps the configure-chosen defaults. +@dataclass +class Config: + name: str + configure: list = field(default_factory=list) + cc: str = "" + cflags: str = "" + ldflags: str = "" + minutes: float = 1.0 + user_settings: str = "" + check: bool = True + prepare: list = field(default_factory=list) + run: list = field(default_factory=list) + +SRCDIR = Path(__file__).resolve().parents[2] +ON_GITHUB = os.environ.get("GITHUB_ACTIONS") == "true" +print_lock = threading.Lock() + +# Fail-fast state: the first failure sets stop_event (under fail_lock, so +# exactly one config is reported as the origin) and kills the other +# workers' in-flight process groups. +stop_event = threading.Event() +fail_lock = threading.Lock() +live_procs = set() +procs_lock = threading.Lock() + + +def abort_others(): + # Every subprocess starts its own session, so killing the process + # group takes down the whole make/test tree under it. + with procs_lock: + procs = list(live_procs) + for p in procs: + try: + os.killpg(p.pid, signal.SIGTERM) + except (ProcessLookupError, PermissionError): + pass + + +def nproc(): + # Like nproc(1): CPUs usable by this process, falling back to all online. + try: + return len(os.sched_getaffinity(0)) + except AttributeError: + return os.cpu_count() or 1 + + +def load_configs(opts, error): + try: + if opts.json == "-": + entries = json.load(sys.stdin) + else: + entries = json.loads(Path(opts.json).read_text()) + except (OSError, ValueError) as e: + error(f"{opts.json}: {e}") + if not isinstance(entries, list): + error(f"{opts.json}: expected a JSON list of config objects") + configs = [] + for entry in entries: + if not isinstance(entry, dict): + error(f"{opts.json}: config entries must be objects: {entry!r}") + unknown = set(entry) - {"name", "configure", "cc", "cflags", + "ldflags", "minutes", "user_settings", + "check", "prepare", "run", "comment"} + if unknown: + error(f"{opts.json}: unknown key(s) in {entry.get('name', entry)!r}: " + f"{' '.join(sorted(unknown))}") + name = entry.get("name") + if not isinstance(name, str) or not name or "/" in name: + error(f"{opts.json}: every config needs a \"name\" usable as a " + f"directory suffix: {entry!r}") + if any(cfg.name == name for cfg in configs): + error(f"{opts.json}: duplicate config name {name!r}") + minutes = entry.get("minutes", 1.0) + if isinstance(minutes, bool) or not isinstance(minutes, (int, float)) \ + or minutes < 0: + error(f"{opts.json}: \"minutes\" must be a non-negative number " + f"in {name!r}") + user_settings = entry.get("user_settings", "") + if not isinstance(user_settings, str): + error(f"{opts.json}: \"user_settings\" must be a path string " + f"in {name!r}") + check = entry.get("check", True) + if not isinstance(check, bool): + error(f"{opts.json}: \"check\" must be a boolean in {name!r}") + cc = entry.get("cc", opts.cc or "") + if not isinstance(cc, str): + error(f"{opts.json}: \"cc\" must be a string in {name!r}") + for key in ("prepare", "run"): + cmds = entry.get(key, []) + if not (isinstance(cmds, list) + and all(isinstance(cmd, list) and cmd + and all(isinstance(a, str) for a in cmd) + for cmd in cmds)): + error(f"{opts.json}: \"{key}\" must be a list of argv lists " + f"in {name!r}") + configs.append(Config(name, list(entry.get("configure", [])), cc, + entry.get("cflags", opts.cflags), + entry.get("ldflags", opts.ldflags), + float(minutes), user_settings, check, + list(entry.get("prepare", [])), + list(entry.get("run", [])))) + if not configs: + error(f"{opts.json}: no configs") + return configs + + +def privatize_dirs(bdir, dirs): + # Replace build-tree symlinks into the source tree with private + # per-build-dir copies: tests that write into these directories would + # otherwise write through the symlink into the shared source tree and + # race with the other parallel checks. Runs after the build steps so + # that build rules which (re)create the symlinks have already run. + for name in dirs: + d = bdir / name + if d.is_symlink(): + d.unlink() + shutil.copytree(SRCDIR / name, d, symlinks=True) + + +def dump(title, path): + print(f"::group::{title}" if ON_GITHUB else f"==== {title} ====") + try: + sys.stdout.write(path.read_text(errors="replace")) + except OSError as e: + print(e) + if ON_GITHUB: + print("::endgroup::") + sys.stdout.flush() + + +def run_config(cfg, opts): + if opts.fail_fast and stop_event.is_set(): + return "aborted", 0.0 + bdir = SRCDIR / f"build-{cfg.name}" + if bdir.exists(): + shutil.rmtree(bdir) + bdir.mkdir() + configure = [str(SRCDIR / "configure")] + cfg.configure + if cfg.cc: + configure.append(f"CC={cfg.cc}") + flags = [f"CFLAGS={cfg.cflags}"] if cfg.cflags else [] + flags += [f"LDFLAGS={cfg.ldflags}"] if cfg.ldflags else [] + make = ["make", f"-j{opts.jobs}"] + flags + steps = [] + if cfg.user_settings: + # Staged before configure; --enable-usersettings builds pick it up + # from the build dir via the default include path. + steps.append((f"stage {cfg.user_settings}", + lambda: shutil.copy(SRCDIR / cfg.user_settings, + bdir / "user_settings.h"))) + steps += [(" ".join(cmd), cmd) for cmd in cfg.prepare] + steps += [("configure", configure), ("make", make)] + if cfg.check: + steps += [ + # Prebuild the check programs without running any tests so + # "make check" below is pure test execution. + ("make check TESTS=", make + ["check", "TESTS="]), + ("private dirs", lambda: privatize_dirs(bdir, opts.private_dir)), + ("make check", ["make"] + flags + ["check"]), + ] + steps += [(" ".join(cmd), cmd) for cmd in cfg.run] + failed = None + start = time.monotonic() + log = bdir / "make-check.log" + with open(log, "w") as logf: + for step, cmd in steps: + if opts.fail_fast and stop_event.is_set(): + failed = "aborted" + break + if callable(cmd): + cmd() + continue + print(f"+ {' '.join(cmd)}", file=logf, flush=True) + # stdin=DEVNULL so a test that reads stdin sees EOF (as in CI) + # instead of blocking forever on an interactive/socket stdin. + proc = subprocess.Popen(cmd, cwd=bdir, stdout=logf, + stderr=subprocess.STDOUT, + stdin=subprocess.DEVNULL, + start_new_session=True) + with procs_lock: + live_procs.add(proc) + try: + rc = proc.wait() + finally: + with procs_lock: + live_procs.discard(proc) + if rc != 0: + if opts.fail_fast: + # The first failure wins; any nonzero exit after the + # abort began was most likely our SIGTERM. + with fail_lock: + failed = "aborted" if stop_event.is_set() else step + stop_event.set() + if failed != "aborted": + abort_others() + else: + failed = step + break + minutes = (time.monotonic() - start) / 60 + with print_lock: + if failed == "aborted": + print(f"{cfg.name}: aborted (fail-fast) [{minutes:.1f} min]") + sys.stdout.flush() + else: + verdict = f"FAIL ({failed})" if failed else "pass" + dump(f"{cfg.name}: {verdict} [{minutes:.1f} min]", log) + if failed == "configure": + dump(f"{cfg.name}: config.log", bdir / "config.log") + elif failed == "make check": + dump(f"{cfg.name}: test-suite.log", bdir / "test-suite.log") + return failed, minutes + + +def summarize(results, wall_min, cpu_min, nthreads): + lines = ["| Config | Result | Minutes |", "|---|---|---|"] + for cfg, failed, minutes in results: + if failed == "aborted": + ok = ":heavy_minus_sign: aborted (fail-fast)" + elif failed: + ok = f":x: FAIL ({failed})" + else: + ok = ":white_check_mark: pass" + lines.append(f"| {cfg.name} | {ok} | {minutes:.1f} |") + # Two views of how efficiently the pool used the machine: thread + # occupancy is the time the workers spent running configs out of the + # thread-minutes available (a long config left for last idles the other + # workers and drags it down); CPU utilization is the CPU time the build + # and test children actually consumed out of the CPU-minutes available + # (too-shallow make -j and serial test phases show up here). + busy_min = sum(minutes for _, _, minutes in results) + ncpu = nproc() + lines += [ + "", + f"{len(results)} configs in {wall_min:.1f} min on {nthreads} " + f"threads / {ncpu} CPUs: " + f"thread occupancy {100 * busy_min / (wall_min * nthreads):.0f}% " + f"({busy_min:.1f} of {wall_min * nthreads:.1f} thread-min), " + f"CPU utilization {100 * cpu_min / (wall_min * ncpu):.0f}% " + f"({cpu_min:.1f} of {wall_min * ncpu:.1f} CPU-min)", + ] + table = "\n".join(lines) + print(table) + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a") as f: + print(f"### make check\n\n{table}", file=f) + + +def main(): + p = argparse.ArgumentParser( + description="Build and make check every configuration from a JSON " + "file in its own out-of-tree build directory, in " + "parallel.") + p.add_argument("json", metavar="CONFIGS.json", + help="JSON list of configs (see the script header for " + "the format), or - for stdin") + p.add_argument("configs", nargs="*", metavar="NAME", + help="configs to run (default: all)") + p.add_argument("--list", action="store_true", help="list configs") + p.add_argument("--jobs", type=int, default=2, + help="make -j per config (default: 2)") + p.add_argument("--threads", type=int, default=nproc(), + help="worker threads; each takes the next pending config " + "when it is free (default: nproc)") + p.add_argument("--shard", metavar="K/N", + help="run only the K-th (1-based) of N shards; configs " + "are dealt to shards greedily by descending " + "\"minutes\" so the shards' totals come out even") + p.add_argument("--fail-fast", action=argparse.BooleanOptionalAction, + default=True, + help="abort everything after the first failing config: " + "pending configs are skipped and in-flight ones " + "killed (--no-fail-fast runs everything and " + "reports every failure)") + p.add_argument("--cc", default="ccache gcc" if shutil.which("ccache") + else None, + help="compiler passed to configure as CC= for configs " + "that do not set their own \"cc\"") + p.add_argument("--cflags", default="", + help="CFLAGS for configs that do not set their own") + p.add_argument("--ldflags", default="", + help="LDFLAGS for configs that do not set their own") + p.add_argument("--private-dir", action="append", default=[], + metavar="DIR", + help="give each build dir a private copy of this " + "symlinked source directory before make check, for " + "tests that write into it (repeatable)") + opts = p.parse_args() + + all_configs = load_configs(opts, p.error) + selected = all_configs + if opts.configs: + by_name = {cfg.name: cfg for cfg in all_configs} + unknown = [n for n in opts.configs if n not in by_name] + if unknown: + p.error(f"unknown config(s): {' '.join(unknown)}") + selected = [by_name[n] for n in opts.configs] + + # Longest first, so the heavyweights never straggle on an otherwise + # idle machine. Stable: configs without "minutes" keep list order. + selected = sorted(selected, key=lambda cfg: -cfg.minutes) + if opts.shard: + try: + k, n = map(int, opts.shard.split("/")) + except ValueError: + k = n = 0 + if not 1 <= k <= n: + p.error(f"--shard: expected K/N with 1 <= K <= N, " + f"got {opts.shard!r}") + # Greedy multiway partition: longest first into the least-loaded + # shard. Deterministic, and with honest "minutes" within ~the + # longest config of optimal. + shards, loads = [[] for _ in range(n)], [0.0] * n + for cfg in selected: + i = loads.index(min(loads)) + shards[i].append(cfg) + loads[i] += cfg.minutes + selected = shards[k - 1] + + if opts.list: + for cfg in selected: + print(f"{cfg.name} [{cfg.minutes:g} min]: " + f"{' '.join(cfg.configure)}") + return 0 + if not selected: + print(f"shard {opts.shard}: no configs to run") + return 0 + + if not (SRCDIR / "configure").exists(): + subprocess.run(["./autogen.sh"], cwd=SRCDIR, check=True) + + nthreads = max(1, min(opts.threads, len(selected))) + wall_start = time.monotonic() + cpu_start = os.times() + with ThreadPoolExecutor(max_workers=nthreads) as pool: + results = [(cfg, failed, minutes) for cfg, (failed, minutes) + in zip(selected, pool.map( + lambda cfg: run_config(cfg, opts), selected))] + wall_min = (time.monotonic() - wall_start) / 60 + cpu_end = os.times() + # os.times() child counters cover the waited-for configure/make + # subprocesses of every worker thread. + cpu_min = (cpu_end.children_user - cpu_start.children_user + + cpu_end.children_system - cpu_start.children_system) / 60 + summarize(results, wall_min, cpu_min, nthreads) + failed = [cfg.name for cfg, failure, _ in results + if failure and failure != "aborted"] + aborted = sum(1 for _, failure, _ in results if failure == "aborted") + if failed or aborted: + msg = f"make check failed for: {' '.join(failed)}" if failed \ + else "aborted without a recorded failure" + if aborted: + msg += f" ({aborted} config(s) aborted by fail-fast)" + print(f"::error::{msg}" if ON_GITHUB else msg) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/async.yml b/.github/workflows/async.yml index 0ed513a21c..d507a378df 100644 --- a/.github/workflows/async.yml +++ b/.github/workflows/async.yml @@ -14,35 +14,89 @@ concurrency: # END OF COMMON SECTION jobs: + # All former runner-per-config matrix entries build on one runner via + # .github/scripts/parallel-make-check.py (see os-check.yml for the full + # pattern): each config in its own out-of-tree ("VPATH") build directory + # off one checkout/autogen, checks on a pool of one-per-CPU worker + # threads, longest first. bubblewrap gives every test script its own + # network namespace so concurrent checks cannot collide on TCP/UDP ports + # (do not set AM_BWRAPPED here - that would disable it). make_check: - strategy: - matrix: - config: [ - # Add new configs here - '--enable-asynccrypt --enable-all --enable-dtls13 --disable-mlkem CFLAGS="-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT"', - '--enable-asynccrypt-sw --enable-ocspstapling --enable-ocspstapling2 --disable-mlkem CFLAGS="-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"', - '--enable-asynccrypt --enable-all --enable-dtls13 CFLAGS="-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT"', - '--enable-asynccrypt-sw --enable-ocspstapling --enable-ocspstapling2 CFLAGS="-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"', - '--enable-ocsp CFLAGS="-DTEST_NONBLOCK_CERTS -pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"', - ] name: make check if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} runs-on: ubuntu-24.04 - # This should be a safe limit for the tests to run. - timeout-minutes: 6 + # Generous for a cold ccache; warm reruns finish in a fraction. + timeout-minutes: 20 steps: - uses: actions/checkout@v4 name: Checkout wolfSSL - - name: Test wolfSSL async - run: | - ./autogen.sh - ./configure ${{ matrix.config }} - make check + - name: Install dependencies + uses: ./.github/actions/install-apt-deps + with: + packages: autoconf automake libtool build-essential bubblewrap - - name: Print errors - if: ${{ failure() }} + # ccache via the cross-platform composite; the script passes the + # compiler to configure as CC="ccache gcc" (or a per-config "cc"). + - name: Set up ccache + uses: ./.github/actions/ccache-setup + with: + workflow-id: async + max-size: 250M + + # Ubuntu 24.04 can restrict unprivileged user namespaces via AppArmor, + # which would stop the test scripts from re-execing under + # bwrap --unshare-net (their port-isolation mechanism). + - name: Allow unprivileged user namespaces (for bwrap) + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true + + # The JSON list below is the former runner-per-config matrix; add new + # configs as new entries (a "comment" key is allowed for notes). + # "minutes" is the expected duration driving longest-first scheduling: + # take it from the Minutes column of a previous run's step summary, or + # omit it for a new config (defaults to 1) and refresh later. The list + # is kept sorted by minutes for readability, but the schedule sorts by + # the values, not list order. + - name: Build and make check all configs (parallel, out-of-tree) run: | - if [ -f test-suite.log ] ; then - cat test-suite.log - fi + cat > "$RUNNER_TEMP/async-configs.json" <<'EOF' + [ + {"name": "asynccrypt-all-no-mlkem", "minutes": 3, + "configure": ["--enable-asynccrypt", "--enable-all", + "--enable-dtls13", "--disable-mlkem", + "CFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT"]}, + {"name": "asynccrypt-all", "minutes": 3, + "configure": ["--enable-asynccrypt", "--enable-all", + "--enable-dtls13", + "CFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT"]}, + {"name": "asynccrypt-sw-ocspstapling-no-mlkem", "minutes": 1.5, + "configure": ["--enable-asynccrypt-sw", "--enable-ocspstapling", + "--enable-ocspstapling2", "--disable-mlkem", + "CFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"]}, + {"name": "asynccrypt-sw-ocspstapling", "minutes": 1.5, + "configure": ["--enable-asynccrypt-sw", "--enable-ocspstapling", + "--enable-ocspstapling2", + "CFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"]}, + {"name": "ocsp-nonblock-certs", "minutes": 1.5, + "configure": ["--enable-ocsp", + "CFLAGS=-DTEST_NONBLOCK_CERTS -pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"]} + ] + EOF + .github/scripts/parallel-make-check.py \ + --private-dir=certs \ + "$RUNNER_TEMP/async-configs.json" + + - name: ccache stats + if: always() + run: ccache -s || true + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: async-logs + path: | + build-*/make-check.log + build-*/test-suite.log + build-*/config.log + if-no-files-found: ignore diff --git a/.github/workflows/check-headers.yml b/.github/workflows/check-headers.yml index dbc3aa6151..17fc0bc33f 100644 --- a/.github/workflows/check-headers.yml +++ b/.github/workflows/check-headers.yml @@ -23,6 +23,8 @@ permissions: jobs: check: + # Only run from the wolfssl org to avoid burning forks' CI minutes. + if: github.repository_owner == 'wolfssl' runs-on: ubuntu-24.04 timeout-minutes: 10 steps: diff --git a/.github/workflows/check-source-text.yml b/.github/workflows/check-source-text.yml index 81d11cc72f..fb6370489e 100644 --- a/.github/workflows/check-source-text.yml +++ b/.github/workflows/check-source-text.yml @@ -28,6 +28,8 @@ permissions: jobs: check: + # Only run from the wolfssl org to avoid burning forks' CI minutes. + if: github.repository_owner == 'wolfssl' runs-on: ubuntu-24.04 timeout-minutes: 5 steps: diff --git a/.github/workflows/cryptocb-only.yml b/.github/workflows/cryptocb-only.yml index 5aedf4c873..a6d6ac4254 100644 --- a/.github/workflows/cryptocb-only.yml +++ b/.github/workflows/cryptocb-only.yml @@ -14,98 +14,222 @@ concurrency: # END OF COMMON SECTION jobs: + # All former runner-per-config matrix entries build on one runner via + # .github/scripts/parallel-make-check.py (see os-check.yml for the full + # pattern): each config in its own out-of-tree ("VPATH") build directory + # off one checkout/autogen, checks on a pool of one-per-CPU worker + # threads, longest first. bubblewrap gives every test script its own + # network namespace so concurrent checks cannot collide on TCP/UDP ports + # (do not set AM_BWRAPPED here - that would disable it). make_check: - strategy: - fail-fast: false - matrix: - include: - # WOLF_CRYPTO_CB_ONLY_ECC: strips software ECC; swdev provides the - # software path via cryptocb. FP_ECC / ECCSI / SAKKE / deterministic-k - # test / OPENSSL_EXTRA compat layer all reference stripped primitives - # directly, so they stay off. - - name: ECC - cppflags: -DWOLF_CRYPTO_CB_ONLY_ECC - # WOLF_CRYPTO_CB_ONLY_RSA: strips software RSA; swdev provides the - # software path via cryptocb. - - name: RSA - cppflags: -DWOLF_CRYPTO_CB_ONLY_RSA - # WOLF_CRYPTO_CB_ONLY_SHA256: strips software SHA-256; swdev provides - # the software path via cryptocb. - - name: SHA256 - cppflags: -DWOLF_CRYPTO_CB_ONLY_SHA256 - # WOLF_CRYPTO_CB_ONLY_SHA512: strips software SHA-512 family (SHA-384, - # SHA-512/224, SHA-512/256, SHA-512); swdev handles every variant - # explicitly via cryptocb. - - name: SHA512 - cppflags: -DWOLF_CRYPTO_CB_ONLY_SHA512 - # Same as SHA512 but tells swdev to refuse the SHA-384 / SHA-512/224 / - # SHA-512/256 variant callbacks (WOLFSSL_SWDEV_SHA512_GENERAL_ONLY). That - # forces the cryptocb dispatcher's fallback-to-plain-SHA-512-with- - # truncation path. The SHA512 entry above instead has swdev handle - # every variant end-to-end, so the dispatcher fallback is otherwise - # uncovered. - - name: SHA512_via_general - cppflags: -DWOLF_CRYPTO_CB_ONLY_SHA512 -DWOLFSSL_SWDEV_SHA512_GENERAL_ONLY - # WOLF_CRYPTO_CB_ONLY_AES: strips software AES; swdev provides the - # software path via cryptocb. - - name: AES - cppflags: -DWOLF_CRYPTO_CB_ONLY_AES - # Same as AES but tells swdev to refuse AES-GCM (SWDEV_AES_ONLYECB). - # That forces the parent's CB_ONLY_AES host-side GCM software path: - # GHASH runs on the host while AES-CTR blocks dispatch back through - # cryptocb ECB. The AES entry above instead has swdev handle GCM - # end-to-end, so the host-side GCM path is otherwise uncovered. - - name: AES_GCM_via_ECB - cppflags: -DWOLF_CRYPTO_CB_ONLY_AES -DSWDEV_AES_ONLYECB - # All four ONLY_* macros at once: every supported software primitive - # is stripped and dispatched through cryptocb. Catches any cross- - # algorithm call that a single-strip entry would still resolve via - # the remaining software paths. - - name: ALL - cppflags: >- - -DWOLF_CRYPTO_CB_ONLY_ECC -DWOLF_CRYPTO_CB_ONLY_RSA - -DWOLF_CRYPTO_CB_ONLY_SHA256 -DWOLF_CRYPTO_CB_ONLY_SHA512 - -DWOLF_CRYPTO_CB_ONLY_AES - name: make check (${{ matrix.name }}) + name: make check if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} runs-on: ubuntu-24.04 - timeout-minutes: 10 - env: - # Common feature set for every entry. - BASE_CONFIG: >- - --enable-swdev --enable-cryptocb --enable-ecc --enable-rsa --enable-dh - --enable-aesgcm --enable-aesccm --enable-aesctr --enable-aescfb - --enable-aeskeywrap --enable-aessiv --enable-aesofb --enable-aesxts - --enable-camellia --enable-chacha --enable-poly1305 - --enable-sha --enable-sha3 --enable-shake128 --enable-shake256 - --enable-blake2 --enable-blake2s - --enable-hkdf --enable-hashdrbg --enable-hashflags - --enable-curve25519 --enable-ed25519 --enable-curve448 --enable-ed448 - --enable-mlkem --enable-dilithium - --enable-scrypt --enable-pwdbased --enable-pkcs7 --enable-pkcs12 - --enable-certgen --enable-certreq --enable-certext - --enable-keygen --enable-asn=all - --enable-cmac --enable-xchacha - --enable-crl --enable-ocsp --enable-ocspstapling --enable-ocspstapling2 - --enable-dtls --enable-dtls13 --enable-tls13 + # Generous for a cold ccache; warm reruns finish in a fraction. + timeout-minutes: 15 steps: - uses: actions/checkout@v4 name: Checkout wolfSSL - - name: Test wolfSSL - run: | - ./autogen.sh - ./configure $BASE_CONFIG ${{ matrix.extra_config }} CPPFLAGS="${{ matrix.cppflags }}" - make -j 4 - make check + - name: Install dependencies + uses: ./.github/actions/install-apt-deps + with: + packages: autoconf automake libtool build-essential bubblewrap - - name: Print errors - if: ${{ failure() }} + # ccache via the cross-platform composite; the script passes the + # compiler to configure as CC="ccache gcc" (or a per-config "cc"). + - name: Set up ccache + uses: ./.github/actions/ccache-setup + with: + workflow-id: cryptocb-only + max-size: 200M + + # Ubuntu 24.04 can restrict unprivileged user namespaces via AppArmor, + # which would stop the test scripts from re-execing under + # bwrap --unshare-net (their port-isolation mechanism). + - name: Allow unprivileged user namespaces (for bwrap) + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true + + # The JSON list below is the former runner-per-config matrix (the + # shared BASE_CONFIG env is folded into every entry); add new configs + # as new entries. "minutes" drives longest-first scheduling: refresh + # it from the Minutes column of a previous run's step summary. + - name: Build and make check all configs (parallel, out-of-tree) run: | - for file in scripts/*.log - do - if [ -f "$file" ]; then - echo "${file}:" - cat "$file" - fi - done + cat > "$RUNNER_TEMP/cryptocb-only-configs.json" <<'EOF' + [ + {"name": "ecc", "minutes": 2, + "comment": "WOLF_CRYPTO_CB_ONLY_ECC: strips software ECC; swdev provides the software path via cryptocb. FP_ECC / ECCSI / SAKKE / deterministic-k test / OPENSSL_EXTRA compat layer all reference stripped primitives directly, so they stay off.", + "configure": ["--enable-swdev", "--enable-cryptocb", "--enable-ecc", + "--enable-rsa", "--enable-dh", "--enable-aesgcm", + "--enable-aesccm", "--enable-aesctr", "--enable-aescfb", + "--enable-aeskeywrap", "--enable-aessiv", "--enable-aesofb", + "--enable-aesxts", "--enable-camellia", "--enable-chacha", + "--enable-poly1305", "--enable-sha", "--enable-sha3", + "--enable-shake128", "--enable-shake256", "--enable-blake2", + "--enable-blake2s", "--enable-hkdf", "--enable-hashdrbg", + "--enable-hashflags", "--enable-curve25519", "--enable-ed25519", + "--enable-curve448", "--enable-ed448", "--enable-mlkem", + "--enable-dilithium", "--enable-scrypt", "--enable-pwdbased", + "--enable-pkcs7", "--enable-pkcs12", "--enable-certgen", + "--enable-certreq", "--enable-certext", "--enable-keygen", + "--enable-asn=all", "--enable-cmac", "--enable-xchacha", + "--enable-crl", "--enable-ocsp", "--enable-ocspstapling", + "--enable-ocspstapling2", "--enable-dtls", "--enable-dtls13", + "--enable-tls13", "CPPFLAGS=-DWOLF_CRYPTO_CB_ONLY_ECC"]}, + {"name": "rsa", "minutes": 2, + "comment": "WOLF_CRYPTO_CB_ONLY_RSA: strips software RSA; swdev provides the software path via cryptocb.", + "configure": ["--enable-swdev", "--enable-cryptocb", "--enable-ecc", + "--enable-rsa", "--enable-dh", "--enable-aesgcm", + "--enable-aesccm", "--enable-aesctr", "--enable-aescfb", + "--enable-aeskeywrap", "--enable-aessiv", "--enable-aesofb", + "--enable-aesxts", "--enable-camellia", "--enable-chacha", + "--enable-poly1305", "--enable-sha", "--enable-sha3", + "--enable-shake128", "--enable-shake256", "--enable-blake2", + "--enable-blake2s", "--enable-hkdf", "--enable-hashdrbg", + "--enable-hashflags", "--enable-curve25519", "--enable-ed25519", + "--enable-curve448", "--enable-ed448", "--enable-mlkem", + "--enable-dilithium", "--enable-scrypt", "--enable-pwdbased", + "--enable-pkcs7", "--enable-pkcs12", "--enable-certgen", + "--enable-certreq", "--enable-certext", "--enable-keygen", + "--enable-asn=all", "--enable-cmac", "--enable-xchacha", + "--enable-crl", "--enable-ocsp", "--enable-ocspstapling", + "--enable-ocspstapling2", "--enable-dtls", "--enable-dtls13", + "--enable-tls13", "CPPFLAGS=-DWOLF_CRYPTO_CB_ONLY_RSA"]}, + {"name": "sha256", "minutes": 2, + "comment": "WOLF_CRYPTO_CB_ONLY_SHA256: strips software SHA-256; swdev provides the software path via cryptocb.", + "configure": ["--enable-swdev", "--enable-cryptocb", "--enable-ecc", + "--enable-rsa", "--enable-dh", "--enable-aesgcm", + "--enable-aesccm", "--enable-aesctr", "--enable-aescfb", + "--enable-aeskeywrap", "--enable-aessiv", "--enable-aesofb", + "--enable-aesxts", "--enable-camellia", "--enable-chacha", + "--enable-poly1305", "--enable-sha", "--enable-sha3", + "--enable-shake128", "--enable-shake256", "--enable-blake2", + "--enable-blake2s", "--enable-hkdf", "--enable-hashdrbg", + "--enable-hashflags", "--enable-curve25519", "--enable-ed25519", + "--enable-curve448", "--enable-ed448", "--enable-mlkem", + "--enable-dilithium", "--enable-scrypt", "--enable-pwdbased", + "--enable-pkcs7", "--enable-pkcs12", "--enable-certgen", + "--enable-certreq", "--enable-certext", "--enable-keygen", + "--enable-asn=all", "--enable-cmac", "--enable-xchacha", + "--enable-crl", "--enable-ocsp", "--enable-ocspstapling", + "--enable-ocspstapling2", "--enable-dtls", "--enable-dtls13", + "--enable-tls13", "CPPFLAGS=-DWOLF_CRYPTO_CB_ONLY_SHA256"]}, + {"name": "sha512", "minutes": 2, + "comment": "WOLF_CRYPTO_CB_ONLY_SHA512: strips software SHA-512 family (SHA-384, SHA-512/224, SHA-512/256, SHA-512); swdev handles every variant explicitly via cryptocb.", + "configure": ["--enable-swdev", "--enable-cryptocb", "--enable-ecc", + "--enable-rsa", "--enable-dh", "--enable-aesgcm", + "--enable-aesccm", "--enable-aesctr", "--enable-aescfb", + "--enable-aeskeywrap", "--enable-aessiv", "--enable-aesofb", + "--enable-aesxts", "--enable-camellia", "--enable-chacha", + "--enable-poly1305", "--enable-sha", "--enable-sha3", + "--enable-shake128", "--enable-shake256", "--enable-blake2", + "--enable-blake2s", "--enable-hkdf", "--enable-hashdrbg", + "--enable-hashflags", "--enable-curve25519", "--enable-ed25519", + "--enable-curve448", "--enable-ed448", "--enable-mlkem", + "--enable-dilithium", "--enable-scrypt", "--enable-pwdbased", + "--enable-pkcs7", "--enable-pkcs12", "--enable-certgen", + "--enable-certreq", "--enable-certext", "--enable-keygen", + "--enable-asn=all", "--enable-cmac", "--enable-xchacha", + "--enable-crl", "--enable-ocsp", "--enable-ocspstapling", + "--enable-ocspstapling2", "--enable-dtls", "--enable-dtls13", + "--enable-tls13", "CPPFLAGS=-DWOLF_CRYPTO_CB_ONLY_SHA512"]}, + {"name": "sha512-via-general", "minutes": 2, + "comment": "Same as sha512 but tells swdev to refuse the SHA-384 / SHA-512/224 / SHA-512/256 variant callbacks (WOLFSSL_SWDEV_SHA512_GENERAL_ONLY). That forces the cryptocb dispatcher's fallback-to-plain-SHA-512-with-truncation path. The sha512 entry above instead has swdev handle every variant end-to-end, so the dispatcher fallback is otherwise uncovered.", + "configure": ["--enable-swdev", "--enable-cryptocb", "--enable-ecc", + "--enable-rsa", "--enable-dh", "--enable-aesgcm", + "--enable-aesccm", "--enable-aesctr", "--enable-aescfb", + "--enable-aeskeywrap", "--enable-aessiv", "--enable-aesofb", + "--enable-aesxts", "--enable-camellia", "--enable-chacha", + "--enable-poly1305", "--enable-sha", "--enable-sha3", + "--enable-shake128", "--enable-shake256", "--enable-blake2", + "--enable-blake2s", "--enable-hkdf", "--enable-hashdrbg", + "--enable-hashflags", "--enable-curve25519", "--enable-ed25519", + "--enable-curve448", "--enable-ed448", "--enable-mlkem", + "--enable-dilithium", "--enable-scrypt", "--enable-pwdbased", + "--enable-pkcs7", "--enable-pkcs12", "--enable-certgen", + "--enable-certreq", "--enable-certext", "--enable-keygen", + "--enable-asn=all", "--enable-cmac", "--enable-xchacha", + "--enable-crl", "--enable-ocsp", "--enable-ocspstapling", + "--enable-ocspstapling2", "--enable-dtls", "--enable-dtls13", + "--enable-tls13", + "CPPFLAGS=-DWOLF_CRYPTO_CB_ONLY_SHA512 -DWOLFSSL_SWDEV_SHA512_GENERAL_ONLY"]}, + {"name": "aes", "minutes": 2, + "comment": "WOLF_CRYPTO_CB_ONLY_AES: strips software AES; swdev provides the software path via cryptocb.", + "configure": ["--enable-swdev", "--enable-cryptocb", "--enable-ecc", + "--enable-rsa", "--enable-dh", "--enable-aesgcm", + "--enable-aesccm", "--enable-aesctr", "--enable-aescfb", + "--enable-aeskeywrap", "--enable-aessiv", "--enable-aesofb", + "--enable-aesxts", "--enable-camellia", "--enable-chacha", + "--enable-poly1305", "--enable-sha", "--enable-sha3", + "--enable-shake128", "--enable-shake256", "--enable-blake2", + "--enable-blake2s", "--enable-hkdf", "--enable-hashdrbg", + "--enable-hashflags", "--enable-curve25519", "--enable-ed25519", + "--enable-curve448", "--enable-ed448", "--enable-mlkem", + "--enable-dilithium", "--enable-scrypt", "--enable-pwdbased", + "--enable-pkcs7", "--enable-pkcs12", "--enable-certgen", + "--enable-certreq", "--enable-certext", "--enable-keygen", + "--enable-asn=all", "--enable-cmac", "--enable-xchacha", + "--enable-crl", "--enable-ocsp", "--enable-ocspstapling", + "--enable-ocspstapling2", "--enable-dtls", "--enable-dtls13", + "--enable-tls13", "CPPFLAGS=-DWOLF_CRYPTO_CB_ONLY_AES"]}, + {"name": "aes-gcm-via-ecb", "minutes": 2, + "comment": "Same as aes but tells swdev to refuse AES-GCM (SWDEV_AES_ONLYECB). That forces the parent's CB_ONLY_AES host-side GCM software path: GHASH runs on the host while AES-CTR blocks dispatch back through cryptocb ECB. The aes entry instead has swdev handle GCM end-to-end, so the host-side GCM path is otherwise uncovered.", + "configure": ["--enable-swdev", "--enable-cryptocb", "--enable-ecc", + "--enable-rsa", "--enable-dh", "--enable-aesgcm", + "--enable-aesccm", "--enable-aesctr", "--enable-aescfb", + "--enable-aeskeywrap", "--enable-aessiv", "--enable-aesofb", + "--enable-aesxts", "--enable-camellia", "--enable-chacha", + "--enable-poly1305", "--enable-sha", "--enable-sha3", + "--enable-shake128", "--enable-shake256", "--enable-blake2", + "--enable-blake2s", "--enable-hkdf", "--enable-hashdrbg", + "--enable-hashflags", "--enable-curve25519", "--enable-ed25519", + "--enable-curve448", "--enable-ed448", "--enable-mlkem", + "--enable-dilithium", "--enable-scrypt", "--enable-pwdbased", + "--enable-pkcs7", "--enable-pkcs12", "--enable-certgen", + "--enable-certreq", "--enable-certext", "--enable-keygen", + "--enable-asn=all", "--enable-cmac", "--enable-xchacha", + "--enable-crl", "--enable-ocsp", "--enable-ocspstapling", + "--enable-ocspstapling2", "--enable-dtls", "--enable-dtls13", + "--enable-tls13", + "CPPFLAGS=-DWOLF_CRYPTO_CB_ONLY_AES -DSWDEV_AES_ONLYECB"]}, + {"name": "all", "minutes": 2, + "comment": "All five ONLY_* macros at once: every supported software primitive is stripped and dispatched through cryptocb. Catches any cross-algorithm call that a single-strip entry would still resolve via the remaining software paths.", + "configure": ["--enable-swdev", "--enable-cryptocb", "--enable-ecc", + "--enable-rsa", "--enable-dh", "--enable-aesgcm", + "--enable-aesccm", "--enable-aesctr", "--enable-aescfb", + "--enable-aeskeywrap", "--enable-aessiv", "--enable-aesofb", + "--enable-aesxts", "--enable-camellia", "--enable-chacha", + "--enable-poly1305", "--enable-sha", "--enable-sha3", + "--enable-shake128", "--enable-shake256", "--enable-blake2", + "--enable-blake2s", "--enable-hkdf", "--enable-hashdrbg", + "--enable-hashflags", "--enable-curve25519", "--enable-ed25519", + "--enable-curve448", "--enable-ed448", "--enable-mlkem", + "--enable-dilithium", "--enable-scrypt", "--enable-pwdbased", + "--enable-pkcs7", "--enable-pkcs12", "--enable-certgen", + "--enable-certreq", "--enable-certext", "--enable-keygen", + "--enable-asn=all", "--enable-cmac", "--enable-xchacha", + "--enable-crl", "--enable-ocsp", "--enable-ocspstapling", + "--enable-ocspstapling2", "--enable-dtls", "--enable-dtls13", + "--enable-tls13", + "CPPFLAGS=-DWOLF_CRYPTO_CB_ONLY_ECC -DWOLF_CRYPTO_CB_ONLY_RSA -DWOLF_CRYPTO_CB_ONLY_SHA256 -DWOLF_CRYPTO_CB_ONLY_SHA512 -DWOLF_CRYPTO_CB_ONLY_AES"]} + ] + EOF + .github/scripts/parallel-make-check.py \ + --private-dir=certs \ + "$RUNNER_TEMP/cryptocb-only-configs.json" + + - name: ccache stats + if: always() + run: ccache -s || true + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cryptocb-only-logs + path: | + build-*/make-check.log + build-*/test-suite.log + build-*/config.log + if-no-files-found: ignore diff --git a/.github/workflows/disable-pk-algs.yml b/.github/workflows/disable-pk-algs.yml index 02e5563ace..1517e9ff0b 100644 --- a/.github/workflows/disable-pk-algs.yml +++ b/.github/workflows/disable-pk-algs.yml @@ -14,51 +14,130 @@ concurrency: # END OF COMMON SECTION jobs: + # All former runner-per-config matrix entries build on one runner via + # .github/scripts/parallel-make-check.py (see os-check.yml for the full + # pattern): each config in its own out-of-tree ("VPATH") build directory + # off one checkout/autogen, checks on a pool of one-per-CPU worker + # threads, longest first. bubblewrap gives every test script its own + # network namespace so concurrent checks cannot collide on TCP/UDP ports + # (do not set AM_BWRAPPED here - that would disable it). make_check: - strategy: - matrix: - config: [ - # Add new configs here - '--disable-rsa --disable-dh --disable-ecc --disable-curve25519 --disable-ed25519 --disable-curve448 --disable-ed448 --enable-rsa --enable-dh', - '--disable-rsa --disable-dh --disable-ecc --disable-curve25519 --disable-ed25519 --disable-curve448 --disable-ed448 --enable-ecc', - '--disable-rsa --disable-dh --disable-ecc --disable-curve25519 --disable-ed25519 --disable-curve448 --disable-ed448 --enable-rsa --enable-curve25519', - '--disable-rsa --disable-dh --disable-ecc --disable-curve25519 --disable-ed25519 --disable-curve448 --disable-ed448 --enable-ecc --enable-curve25519', - '--disable-rsa --disable-dh --disable-ecc --disable-curve25519 --disable-ed25519 --disable-curve448 --disable-ed448 --enable-rsa --enable-curve448', - '--disable-rsa --disable-dh --disable-ecc --disable-curve25519 --disable-ed25519 --disable-curve448 --disable-ed448 --enable-ecc --enable-curve448', - '--disable-rsa --disable-dh --disable-ecc --disable-curve25519 --disable-ed25519 --disable-curve448 --disable-ed448 --enable-curve25519 --enable-ed25519', - '--disable-rsa --disable-dh --disable-ecc --disable-curve25519 --disable-ed25519 --disable-curve448 --disable-ed448 --enable-curve448 --enable-ed448', - '-enable-cryptonly --disable-rsa --disable-dh --disable-ecc --disable-curve25519 --disable-ed25519 --disable-curve448 --disable-ed448 --enable-rsa', - '--enable-cryptonly --disable-rsa --disable-dh --disable-ecc --disable-curve25519 --disable-ed25519 --disable-curve448 --disable-ed448 --enable-dh', - '--enable-cryptonly --disable-rsa --disable-dh --disable-ecc --disable-curve25519 --disable-ed25519 --disable-curve448 --disable-ed448 --enable-ecc', - '--enable-cryptonly --disable-rsa --disable-dh --disable-ecc --disable-curve25519 --disable-ed25519 --disable-curve448 --disable-ed448 --enable-curve25519', - '--enable-cryptonly --disable-rsa --disable-dh --disable-ecc --disable-curve25519 --disable-ed25519 --disable-curve448 --disable-ed448 --enable-ed25519', - '--enable-cryptonly --disable-rsa --disable-dh --disable-ecc --disable-curve25519 --disable-ed25519 --disable-curve448 --disable-ed448 --enable-curve448', - '--enable-cryptonly --disable-rsa --disable-dh --disable-ecc --disable-curve25519 --disable-ed25519 --disable-curve448 --disable-ed448 --enable-ed448', - ] name: make check if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} runs-on: ubuntu-24.04 - # This should be a safe limit for the tests to run. - timeout-minutes: 6 + # Generous for a cold ccache; warm reruns finish in a fraction. + timeout-minutes: 15 steps: - uses: actions/checkout@v4 name: Checkout wolfSSL - - name: Test wolfSSL - run: | - ./autogen.sh - ./configure ${{ matrix.config }} - make -j 4 - make check + - name: Install dependencies + uses: ./.github/actions/install-apt-deps + with: + packages: autoconf automake libtool build-essential bubblewrap - - name: Print errors - if: ${{ failure() }} + # ccache via the cross-platform composite; the script passes the + # compiler to configure as CC="ccache gcc" (or a per-config "cc"). + - name: Set up ccache + uses: ./.github/actions/ccache-setup + with: + workflow-id: disable-pk-algs + max-size: 150M + + # Ubuntu 24.04 can restrict unprivileged user namespaces via AppArmor, + # which would stop the test scripts from re-execing under + # bwrap --unshare-net (their port-isolation mechanism). + - name: Allow unprivileged user namespaces (for bwrap) + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true + + # The JSON list below is the former runner-per-config matrix; add new + # configs as new entries (a "comment" key is allowed for notes). + # "minutes" is the expected duration driving longest-first scheduling: + # take it from the Minutes column of a previous run's step summary, or + # omit it for a new config (defaults to 1) and refresh later. The list + # is kept sorted by minutes for readability, but the schedule sorts by + # the values, not list order. + - name: Build and make check all configs (parallel, out-of-tree) run: | - for file in scripts/*.log - do - if [ -f "$file" ]; then - echo "${file}:" - cat "$file" - echo "========================================================================" - fi - done + cat > "$RUNNER_TEMP/disable-pk-algs-configs.json" <<'EOF' + [ + {"name": "rsa-dh", "minutes": 1.2, + "configure": ["--disable-rsa", "--disable-dh", "--disable-ecc", + "--disable-curve25519", "--disable-ed25519", "--disable-curve448", + "--disable-ed448", "--enable-rsa", "--enable-dh"]}, + {"name": "ecc", "minutes": 1.2, + "configure": ["--disable-rsa", "--disable-dh", "--disable-ecc", + "--disable-curve25519", "--disable-ed25519", "--disable-curve448", + "--disable-ed448", "--enable-ecc"]}, + {"name": "rsa-curve25519", "minutes": 1.2, + "configure": ["--disable-rsa", "--disable-dh", "--disable-ecc", + "--disable-curve25519", "--disable-ed25519", "--disable-curve448", + "--disable-ed448", "--enable-rsa", "--enable-curve25519"]}, + {"name": "ecc-curve25519", "minutes": 1.2, + "configure": ["--disable-rsa", "--disable-dh", "--disable-ecc", + "--disable-curve25519", "--disable-ed25519", "--disable-curve448", + "--disable-ed448", "--enable-ecc", "--enable-curve25519"]}, + {"name": "rsa-curve448", "minutes": 1.2, + "configure": ["--disable-rsa", "--disable-dh", "--disable-ecc", + "--disable-curve25519", "--disable-ed25519", "--disable-curve448", + "--disable-ed448", "--enable-rsa", "--enable-curve448"]}, + {"name": "ecc-curve448", "minutes": 1.2, + "configure": ["--disable-rsa", "--disable-dh", "--disable-ecc", + "--disable-curve25519", "--disable-ed25519", "--disable-curve448", + "--disable-ed448", "--enable-ecc", "--enable-curve448"]}, + {"name": "curve25519-ed25519", "minutes": 1.2, + "configure": ["--disable-rsa", "--disable-dh", "--disable-ecc", + "--disable-curve25519", "--disable-ed25519", "--disable-curve448", + "--disable-ed448", "--enable-curve25519", "--enable-ed25519"]}, + {"name": "curve448-ed448", "minutes": 1.2, + "configure": ["--disable-rsa", "--disable-dh", "--disable-ecc", + "--disable-curve25519", "--disable-ed25519", "--disable-curve448", + "--disable-ed448", "--enable-curve448", "--enable-ed448"]}, + {"name": "cryptonly-rsa", "minutes": 0.8, + "configure": ["-enable-cryptonly", "--disable-rsa", "--disable-dh", + "--disable-ecc", "--disable-curve25519", "--disable-ed25519", + "--disable-curve448", "--disable-ed448", "--enable-rsa"]}, + {"name": "cryptonly-dh", "minutes": 0.8, + "configure": ["--enable-cryptonly", "--disable-rsa", "--disable-dh", + "--disable-ecc", "--disable-curve25519", "--disable-ed25519", + "--disable-curve448", "--disable-ed448", "--enable-dh"]}, + {"name": "cryptonly-ecc", "minutes": 0.8, + "configure": ["--enable-cryptonly", "--disable-rsa", "--disable-dh", + "--disable-ecc", "--disable-curve25519", "--disable-ed25519", + "--disable-curve448", "--disable-ed448", "--enable-ecc"]}, + {"name": "cryptonly-curve25519", "minutes": 0.8, + "configure": ["--enable-cryptonly", "--disable-rsa", "--disable-dh", + "--disable-ecc", "--disable-curve25519", "--disable-ed25519", + "--disable-curve448", "--disable-ed448", "--enable-curve25519"]}, + {"name": "cryptonly-ed25519", "minutes": 0.8, + "configure": ["--enable-cryptonly", "--disable-rsa", "--disable-dh", + "--disable-ecc", "--disable-curve25519", "--disable-ed25519", + "--disable-curve448", "--disable-ed448", "--enable-ed25519"]}, + {"name": "cryptonly-curve448", "minutes": 0.8, + "configure": ["--enable-cryptonly", "--disable-rsa", "--disable-dh", + "--disable-ecc", "--disable-curve25519", "--disable-ed25519", + "--disable-curve448", "--disable-ed448", "--enable-curve448"]}, + {"name": "cryptonly-ed448", "minutes": 0.8, + "configure": ["--enable-cryptonly", "--disable-rsa", "--disable-dh", + "--disable-ecc", "--disable-curve25519", "--disable-ed25519", + "--disable-curve448", "--disable-ed448", "--enable-ed448"]} + ] + EOF + .github/scripts/parallel-make-check.py \ + --private-dir=certs \ + "$RUNNER_TEMP/disable-pk-algs-configs.json" + + - name: ccache stats + if: always() + run: ccache -s || true + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: disable-pk-algs-logs + path: | + build-*/make-check.log + build-*/test-suite.log + build-*/config.log + if-no-files-found: ignore diff --git a/.github/workflows/libspdm.yml b/.github/workflows/libspdm.yml index 0f0ed84756..f8b606ee79 100644 --- a/.github/workflows/libspdm.yml +++ b/.github/workflows/libspdm.yml @@ -19,8 +19,9 @@ jobs: if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} # Just to keep it the same as the testing target runs-on: ubuntu-24.04 - # This should be a safe limit for the tests to run. - timeout-minutes: 4 + # 4 min had no margin for a full --enable-all wolfSSL build on a loaded + # runner and flakily cancelled (job timeouts report as "cancelled"). + timeout-minutes: 10 steps: - name: Build wolfSSL uses: wolfSSL/actions-build-autotools-project@v1 @@ -48,8 +49,9 @@ jobs: name: ${{ matrix.ref }} if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} runs-on: ubuntu-24.04 - # This should be a safe limit for the tests to run. - timeout-minutes: 4 + # 4 min had no margin for a full --enable-all wolfSSL build on a loaded + # runner and flakily cancelled (job timeouts report as "cancelled"). + timeout-minutes: 10 needs: build_wolfssl steps: - name: Download lib diff --git a/.github/workflows/macos-apple-native-cert-validation.yml b/.github/workflows/macos-apple-native-cert-validation.yml deleted file mode 100644 index c431ee64a5..0000000000 --- a/.github/workflows/macos-apple-native-cert-validation.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: MacOS apple native cert validation tests - -# START OF COMMON SECTION -on: - push: - branches: [ 'release/**' ] - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - branches: [ '*' ] - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION - -jobs: - make_check: - if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} - runs-on: macos-latest - # This should be a safe limit for the tests to run. - timeout-minutes: 5 - steps: - - name: Build and configure wolfSSL - uses: wolfSSL/actions-build-autotools-project@v1 - with: - configure: CFLAGS='-DWOLFSSL_APPLE_NATIVE_CERT_VALIDATION -DWOLFSSL_TEST_APPLE_NATIVE_CERT_VALIDATION -DRSA_MIN_SIZE=2048 -DNO_WOLFSSL_CIPHER_SUITE_TEST' - diff --git a/.github/workflows/multi-arch.yml b/.github/workflows/multi-arch.yml index f12afd5548..92924e15a6 100644 --- a/.github/workflows/multi-arch.yml +++ b/.github/workflows/multi-arch.yml @@ -14,67 +14,251 @@ concurrency: # END OF COMMON SECTION jobs: + # All former runner-per-config matrix entries build on one runner via + # .github/scripts/parallel-make-check.py (see os-check.yml for the full + # pattern): each config builds in its own out-of-tree ("VPATH") build + # directory off one checkout/autogen, on a pool of one-per-CPU worker + # threads, longest first. my_matrix: - name: Multi-arch test (${{ matrix.ARCH }}, ${{ matrix.opts.name }}) - strategy: - fail-fast: false - matrix: - include: - - HOST: aarch64-linux-gnu - CC: aarch64-linux-gnu-gcc - ARCH: arm64 - EXTRA_OPTS: --enable-sp-asm --enable-armasm - - HOST: arm-linux-gnueabihf - CC: arm-linux-gnueabihf-gcc - ARCH: armhf - EXTRA_OPTS: --enable-sp-asm - - HOST: riscv64-linux-gnu - CC: riscv64-linux-gnu-gcc - ARCH: riscv64 - EXTRA_OPTS: --enable-riscv-asm - # Config to ensure CPUs without Thumb instructions compiles - - HOST: arm-linux-gnueabi - CC: arm-linux-gnueabi-gcc - CFLAGS: -marm -DWOLFSSL_SP_ARM_ARCH=6 - ARCH: armel - EXTRA_OPTS: --enable-sp-asm - opts: - - name: '-O2' - OPT_CFLAGS: '-O2' - - name: '-O2 sp-math' - OPT_CFLAGS: '-O2' - OPT_EXTRA_OPTS: '--enable-sp-math' - - name: '-O1 -UFP_ECC' - OPT_CFLAGS: '-O1 -UFP_ECC' - - name: '-O0' - OPT_CFLAGS: '-O0' - - name: '-Os' - OPT_CFLAGS: '-Os' - - name: '-Ofast' - OPT_CFLAGS: '-Ofast' + name: Multi-arch test if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} runs-on: ubuntu-22.04 - # This should be a safe limit for the tests to run. - timeout-minutes: 10 + # Generous for a cold ccache; warm reruns finish in a fraction. + timeout-minutes: 35 steps: - uses: actions/checkout@v4 - - name: Install Compiler + name: Checkout wolfSSL + + - name: Install dependencies uses: ./.github/actions/install-apt-deps with: - packages: crossbuild-essential-${{ matrix.ARCH }} qemu-user - - name: Build for ${{ matrix.ARCH }} with ${{ matrix.opts.name }} - env: - CC: ${{ matrix.CC }} - CFLAGS: ${{ matrix.CFLAGS }} ${{ matrix.opts.OPT_CFLAGS }} - QEMU_LD_PREFIX: /usr/${{ matrix.HOST }} - run: ./autogen.sh && ./configure --host=${{ matrix.HOST }} --enable-all --disable-examples CPPFLAGS="-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT" ${{ matrix.EXTRA_OPTS }} ${{ matrix.opts.OPT_EXTRA_OPTS }} && make - - name: Print errors - if: ${{ failure() }} + packages: autoconf automake libtool build-essential crossbuild-essential-arm64 crossbuild-essential-armhf crossbuild-essential-riscv64 crossbuild-essential-armel qemu-user + + # ccache via the cross-platform composite; the script passes the + # compiler to configure as CC="ccache gcc" (or a per-config "cc"). + - name: Set up ccache + uses: ./.github/actions/ccache-setup + with: + workflow-id: multi-arch + max-size: 500M + + # NOTE: the old runner-per-config matrix combined an "include" list + # of four architectures with an "opts" axis; GitHub's include-merge + # rules made each arch entry overwrite the previous one, so only the + # last (armel) combinations actually ran. The JSON below restores the + # evidently intended aarch64/armhf/riscv64 x opts coverage alongside + # armel, except riscv64 x sp-math: configure rejects --enable-sp-math + # without SP, and riscv64's --enable-riscv-asm (unlike the other + # arches' --enable-sp-asm) does not bring it in. Cross builds run + # testwolfcrypt transparently under qemu-user (binfmt) with the + # matching QEMU_LD_PREFIX. + - name: Build all configs (parallel, out-of-tree) run: | - if [ -f config.log ] ; then - cat config.log - fi - - name: Run WolfCrypt Tests - env: - QEMU_LD_PREFIX: /usr/${{ matrix.HOST }} - run: ./wolfcrypt/test/testwolfcrypt + cat > "$RUNNER_TEMP/multi-arch-configs.json" <<'EOF' + [ + {"name": "arm64-o0", "minutes": 4, + "cc": "ccache aarch64-linux-gnu-gcc", + "configure": ["--host=aarch64-linux-gnu", "--enable-all", + "--disable-examples", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT", + "--enable-sp-asm", "--enable-armasm", "CFLAGS=-O0"], + "check": false, + "run": [["env", "QEMU_LD_PREFIX=/usr/aarch64-linux-gnu", "./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "armhf-o0", "minutes": 4, + "cc": "ccache arm-linux-gnueabihf-gcc", + "configure": ["--host=arm-linux-gnueabihf", "--enable-all", + "--disable-examples", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT", + "--enable-sp-asm", "CFLAGS=-O0"], + "check": false, + "run": [["env", "QEMU_LD_PREFIX=/usr/arm-linux-gnueabihf", "./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "riscv64-o0", "minutes": 4, + "cc": "ccache riscv64-linux-gnu-gcc", + "configure": ["--host=riscv64-linux-gnu", "--enable-all", + "--disable-examples", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT", + "--enable-riscv-asm", "CFLAGS=-O0"], + "check": false, + "run": [["env", "QEMU_LD_PREFIX=/usr/riscv64-linux-gnu", "./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "armel-o0", "minutes": 4, + "cc": "ccache arm-linux-gnueabi-gcc", + "configure": ["--host=arm-linux-gnueabi", "--enable-all", + "--disable-examples", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT", + "--enable-sp-asm", "CFLAGS=-marm -DWOLFSSL_SP_ARM_ARCH=6 -O0"], + "check": false, + "run": [["env", "QEMU_LD_PREFIX=/usr/arm-linux-gnueabi", "./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "arm64-o1-no-fp-ecc", "minutes": 3, + "cc": "ccache aarch64-linux-gnu-gcc", + "configure": ["--host=aarch64-linux-gnu", "--enable-all", + "--disable-examples", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT", + "--enable-sp-asm", "--enable-armasm", "CFLAGS=-O1 -UFP_ECC"], + "check": false, + "run": [["env", "QEMU_LD_PREFIX=/usr/aarch64-linux-gnu", "./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "arm64-os", "minutes": 3, + "cc": "ccache aarch64-linux-gnu-gcc", + "configure": ["--host=aarch64-linux-gnu", "--enable-all", + "--disable-examples", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT", + "--enable-sp-asm", "--enable-armasm", "CFLAGS=-Os"], + "check": false, + "run": [["env", "QEMU_LD_PREFIX=/usr/aarch64-linux-gnu", "./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "armhf-o1-no-fp-ecc", "minutes": 3, + "cc": "ccache arm-linux-gnueabihf-gcc", + "configure": ["--host=arm-linux-gnueabihf", "--enable-all", + "--disable-examples", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT", + "--enable-sp-asm", "CFLAGS=-O1 -UFP_ECC"], + "check": false, + "run": [["env", "QEMU_LD_PREFIX=/usr/arm-linux-gnueabihf", "./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "armhf-os", "minutes": 3, + "cc": "ccache arm-linux-gnueabihf-gcc", + "configure": ["--host=arm-linux-gnueabihf", "--enable-all", + "--disable-examples", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT", + "--enable-sp-asm", "CFLAGS=-Os"], + "check": false, + "run": [["env", "QEMU_LD_PREFIX=/usr/arm-linux-gnueabihf", "./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "riscv64-o1-no-fp-ecc", "minutes": 3, + "cc": "ccache riscv64-linux-gnu-gcc", + "configure": ["--host=riscv64-linux-gnu", "--enable-all", + "--disable-examples", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT", + "--enable-riscv-asm", "CFLAGS=-O1 -UFP_ECC"], + "check": false, + "run": [["env", "QEMU_LD_PREFIX=/usr/riscv64-linux-gnu", "./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "riscv64-os", "minutes": 3, + "cc": "ccache riscv64-linux-gnu-gcc", + "configure": ["--host=riscv64-linux-gnu", "--enable-all", + "--disable-examples", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT", + "--enable-riscv-asm", "CFLAGS=-Os"], + "check": false, + "run": [["env", "QEMU_LD_PREFIX=/usr/riscv64-linux-gnu", "./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "armel-o1-no-fp-ecc", "minutes": 3, + "cc": "ccache arm-linux-gnueabi-gcc", + "configure": ["--host=arm-linux-gnueabi", "--enable-all", + "--disable-examples", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT", + "--enable-sp-asm", + "CFLAGS=-marm -DWOLFSSL_SP_ARM_ARCH=6 -O1 -UFP_ECC"], + "check": false, + "run": [["env", "QEMU_LD_PREFIX=/usr/arm-linux-gnueabi", "./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "armel-os", "minutes": 3, + "cc": "ccache arm-linux-gnueabi-gcc", + "configure": ["--host=arm-linux-gnueabi", "--enable-all", + "--disable-examples", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT", + "--enable-sp-asm", "CFLAGS=-marm -DWOLFSSL_SP_ARM_ARCH=6 -Os"], + "check": false, + "run": [["env", "QEMU_LD_PREFIX=/usr/arm-linux-gnueabi", "./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "arm64-o2", "minutes": 2.5, + "cc": "ccache aarch64-linux-gnu-gcc", + "configure": ["--host=aarch64-linux-gnu", "--enable-all", + "--disable-examples", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT", + "--enable-sp-asm", "--enable-armasm", "CFLAGS=-O2"], + "check": false, + "run": [["env", "QEMU_LD_PREFIX=/usr/aarch64-linux-gnu", "./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "arm64-o2-sp-math", "minutes": 2.5, + "cc": "ccache aarch64-linux-gnu-gcc", + "configure": ["--host=aarch64-linux-gnu", "--enable-all", + "--disable-examples", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT", + "--enable-sp-asm", "--enable-armasm", "--enable-sp-math", + "CFLAGS=-O2"], + "check": false, + "run": [["env", "QEMU_LD_PREFIX=/usr/aarch64-linux-gnu", "./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "arm64-ofast", "minutes": 2.5, + "cc": "ccache aarch64-linux-gnu-gcc", + "configure": ["--host=aarch64-linux-gnu", "--enable-all", + "--disable-examples", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT", + "--enable-sp-asm", "--enable-armasm", "CFLAGS=-Ofast"], + "check": false, + "run": [["env", "QEMU_LD_PREFIX=/usr/aarch64-linux-gnu", "./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "armhf-o2", "minutes": 2.5, + "cc": "ccache arm-linux-gnueabihf-gcc", + "configure": ["--host=arm-linux-gnueabihf", "--enable-all", + "--disable-examples", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT", + "--enable-sp-asm", "CFLAGS=-O2"], + "check": false, + "run": [["env", "QEMU_LD_PREFIX=/usr/arm-linux-gnueabihf", "./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "armhf-o2-sp-math", "minutes": 2.5, + "cc": "ccache arm-linux-gnueabihf-gcc", + "configure": ["--host=arm-linux-gnueabihf", "--enable-all", + "--disable-examples", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT", + "--enable-sp-asm", "--enable-sp-math", "CFLAGS=-O2"], + "check": false, + "run": [["env", "QEMU_LD_PREFIX=/usr/arm-linux-gnueabihf", "./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "armhf-ofast", "minutes": 2.5, + "cc": "ccache arm-linux-gnueabihf-gcc", + "configure": ["--host=arm-linux-gnueabihf", "--enable-all", + "--disable-examples", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT", + "--enable-sp-asm", "CFLAGS=-Ofast"], + "check": false, + "run": [["env", "QEMU_LD_PREFIX=/usr/arm-linux-gnueabihf", "./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "riscv64-o2", "minutes": 2.5, + "cc": "ccache riscv64-linux-gnu-gcc", + "configure": ["--host=riscv64-linux-gnu", "--enable-all", + "--disable-examples", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT", + "--enable-riscv-asm", "CFLAGS=-O2"], + "check": false, + "run": [["env", "QEMU_LD_PREFIX=/usr/riscv64-linux-gnu", "./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "riscv64-ofast", "minutes": 2.5, + "cc": "ccache riscv64-linux-gnu-gcc", + "configure": ["--host=riscv64-linux-gnu", "--enable-all", + "--disable-examples", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT", + "--enable-riscv-asm", "CFLAGS=-Ofast"], + "check": false, + "run": [["env", "QEMU_LD_PREFIX=/usr/riscv64-linux-gnu", "./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "armel-o2", "minutes": 2.5, + "cc": "ccache arm-linux-gnueabi-gcc", + "configure": ["--host=arm-linux-gnueabi", "--enable-all", + "--disable-examples", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT", + "--enable-sp-asm", "CFLAGS=-marm -DWOLFSSL_SP_ARM_ARCH=6 -O2"], + "check": false, + "run": [["env", "QEMU_LD_PREFIX=/usr/arm-linux-gnueabi", "./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "armel-o2-sp-math", "minutes": 2.5, + "cc": "ccache arm-linux-gnueabi-gcc", + "configure": ["--host=arm-linux-gnueabi", "--enable-all", + "--disable-examples", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT", + "--enable-sp-asm", "--enable-sp-math", + "CFLAGS=-marm -DWOLFSSL_SP_ARM_ARCH=6 -O2"], + "check": false, + "run": [["env", "QEMU_LD_PREFIX=/usr/arm-linux-gnueabi", "./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "armel-ofast", "minutes": 2.5, + "cc": "ccache arm-linux-gnueabi-gcc", + "configure": ["--host=arm-linux-gnueabi", "--enable-all", + "--disable-examples", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFCRYPT_TEST_LINT", + "--enable-sp-asm", "CFLAGS=-marm -DWOLFSSL_SP_ARM_ARCH=6 -Ofast"], + "check": false, + "run": [["env", "QEMU_LD_PREFIX=/usr/arm-linux-gnueabi", "./wolfcrypt/test/testwolfcrypt"]]} + ] + EOF + .github/scripts/parallel-make-check.py \ + "$RUNNER_TEMP/multi-arch-configs.json" + + - name: ccache stats + if: always() + run: ccache -s || true + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: multi-arch-logs + path: | + build-*/make-check.log + build-*/test-suite.log + build-*/config.log + if-no-files-found: ignore diff --git a/.github/workflows/multi-compiler.yml b/.github/workflows/multi-compiler.yml index 422c3c79ef..e25e6ec50a 100644 --- a/.github/workflows/multi-compiler.yml +++ b/.github/workflows/multi-compiler.yml @@ -14,46 +14,94 @@ concurrency: # END OF COMMON SECTION jobs: + # All former runner-per-config matrix entries build on one runner via + # .github/scripts/parallel-make-check.py (see os-check.yml for the full + # pattern): each config builds in its own out-of-tree ("VPATH") build + # directory off one checkout/autogen, on a pool of one-per-CPU worker + # threads, longest first. my_matrix: name: Compiler test - strategy: - fail-fast: false - matrix: - include: - - CC: gcc-9 - CXX: g++-9 - OS: ubuntu-24.04 - - CC: gcc-10 - CXX: g++-10 - OS: ubuntu-24.04 - - CC: gcc-11 - CXX: g++-11 - OS: ubuntu-24.04 - - CC: gcc-12 - CXX: g++-12 - OS: ubuntu-24.04 - - CC: clang-14 - CXX: clang++-14 - OS: ubuntu-24.04 - - CC: clang-19 - CXX: clang++-19 - OS: ubuntu-24.04 if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} - runs-on: ${{ matrix.OS }} - # This should be a safe limit for the tests to run. - timeout-minutes: 4 + runs-on: ubuntu-24.04 + # Generous for a cold ccache; warm reruns finish in a fraction. + timeout-minutes: 15 steps: - uses: actions/checkout@v4 + name: Checkout wolfSSL + - name: Install dependencies uses: ./.github/actions/install-apt-deps with: - packages: ${{ matrix.CC }} - - name: Build - env: - CC: ${{ matrix.CC }} - CXX: ${{ matrix.CXX }} - run: ./autogen.sh && ./configure CFLAGS="-pedantic -Wdeclaration-after-statement -Wnull-dereference" && make && make dist - - name: Show log on errors - if: ${{ failure() }} + packages: autoconf automake libtool build-essential gcc-9 g++-9 gcc-10 g++-10 gcc-11 g++-11 gcc-12 g++-12 clang-14 clang-19 + + # ccache via the cross-platform composite; the script passes the + # compiler to configure as CC="ccache gcc" (or a per-config "cc"). + - name: Set up ccache + uses: ./.github/actions/ccache-setup + with: + workflow-id: multi-compiler + max-size: 500M + + # One entry per compiler (the former one-runner-per-compiler matrix): + # default-config build plus "make dist", with the compiler routed + # through ccache via the per-config "cc" key. CXX rides along as a + # configure argument. + - name: Build all configs (parallel, out-of-tree) run: | - cat config.log + cat > "$RUNNER_TEMP/multi-compiler-configs.json" <<'EOF' + [ + {"name": "gcc-9", "minutes": 1.5, + "cc": "ccache gcc-9", + "configure": ["CXX=g++-9", + "CFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference"], + "check": false, + "run": [["make", "dist"]]}, + {"name": "gcc-10", "minutes": 1.5, + "cc": "ccache gcc-10", + "configure": ["CXX=g++-10", + "CFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference"], + "check": false, + "run": [["make", "dist"]]}, + {"name": "gcc-11", "minutes": 1.5, + "cc": "ccache gcc-11", + "configure": ["CXX=g++-11", + "CFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference"], + "check": false, + "run": [["make", "dist"]]}, + {"name": "gcc-12", "minutes": 1.5, + "cc": "ccache gcc-12", + "configure": ["CXX=g++-12", + "CFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference"], + "check": false, + "run": [["make", "dist"]]}, + {"name": "clang-14", "minutes": 1.5, + "cc": "ccache clang-14", + "configure": ["CXX=clang++-14", + "CFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference"], + "check": false, + "run": [["make", "dist"]]}, + {"name": "clang-19", "minutes": 1.5, + "cc": "ccache clang-19", + "configure": ["CXX=clang++-19", + "CFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference"], + "check": false, + "run": [["make", "dist"]]} + ] + EOF + .github/scripts/parallel-make-check.py \ + "$RUNNER_TEMP/multi-compiler-configs.json" + + - name: ccache stats + if: always() + run: ccache -s || true + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: multi-compiler-logs + path: | + build-*/make-check.log + build-*/test-suite.log + build-*/config.log + if-no-files-found: ignore diff --git a/.github/workflows/no-malloc.yml b/.github/workflows/no-malloc.yml index c35419d519..9aa25bc8e0 100644 --- a/.github/workflows/no-malloc.yml +++ b/.github/workflows/no-malloc.yml @@ -14,34 +14,76 @@ concurrency: # END OF COMMON SECTION jobs: + # All former runner-per-config matrix entries build on one runner via + # .github/scripts/parallel-make-check.py (see os-check.yml for the full + # pattern): each config builds in its own out-of-tree ("VPATH") build + # directory off one checkout/autogen, on a pool of one-per-CPU worker + # threads, longest first. make_check: - strategy: - matrix: - config: [ - # Add new configs here - '--enable-rsa --enable-keygen --disable-dh CFLAGS="-DWOLFSSL_NO_MALLOC -DRSA_MIN_SIZE=1024 -pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"', - '--enable-ecc --enable-rsa --enable-keygen --enable-ed25519 --enable-curve25519 --enable-ed448 --enable-curve448 --enable-mlkem CFLAGS="-DWOLFSSL_NO_MALLOC -pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"', - '--enable-ecc --enable-rsa --enable-keygen --enable-ed25519 --enable-curve25519 --enable-ed448 --enable-curve448 --enable-mlkem --enable-staticmemory CFLAGS="-DWOLFSSL_NO_MALLOC -pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"', - ] name: make check if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} runs-on: ubuntu-24.04 - # This should be a safe limit for the tests to run. - timeout-minutes: 6 + # Generous for a cold ccache; warm reruns finish in a fraction. + timeout-minutes: 10 steps: - uses: actions/checkout@v4 name: Checkout wolfSSL - - name: Test wolfSSL - run: | - ./autogen.sh - ./configure ${{ matrix.config }} - make -j - ./wolfcrypt/test/testwolfcrypt + - name: Install dependencies + uses: ./.github/actions/install-apt-deps + with: + packages: autoconf automake libtool build-essential - - name: Print errors - if: ${{ failure() }} + # ccache via the cross-platform composite; the script passes the + # compiler to configure as CC="ccache gcc" (or a per-config "cc"). + - name: Set up ccache + uses: ./.github/actions/ccache-setup + with: + workflow-id: no-malloc + max-size: 100M + + # The JSON list below is the former runner-per-config matrix. These + # configs never ran the TLS check phase - just the build plus + # testwolfcrypt - hence "check": false with an explicit "run". + - name: Build all configs (parallel, out-of-tree) run: | - if [ -f test-suite.log ] ; then - cat test-suite.log - fi + cat > "$RUNNER_TEMP/no-malloc-configs.json" <<'EOF' + [ + {"name": "rsa-keygen", "minutes": 0.8, + "configure": ["--enable-rsa", "--enable-keygen", "--disable-dh", + "CFLAGS=-DWOLFSSL_NO_MALLOC -DRSA_MIN_SIZE=1024 -pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"], + "check": false, + "run": [["./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "pk-mlkem", "minutes": 0.8, + "configure": ["--enable-ecc", "--enable-rsa", "--enable-keygen", + "--enable-ed25519", "--enable-curve25519", "--enable-ed448", + "--enable-curve448", "--enable-mlkem", + "CFLAGS=-DWOLFSSL_NO_MALLOC -pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"], + "check": false, + "run": [["./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "pk-mlkem-staticmemory", "minutes": 0.8, + "configure": ["--enable-ecc", "--enable-rsa", "--enable-keygen", + "--enable-ed25519", "--enable-curve25519", "--enable-ed448", + "--enable-curve448", "--enable-mlkem", "--enable-staticmemory", + "CFLAGS=-DWOLFSSL_NO_MALLOC -pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"], + "check": false, + "run": [["./wolfcrypt/test/testwolfcrypt"]]} + ] + EOF + .github/scripts/parallel-make-check.py \ + "$RUNNER_TEMP/no-malloc-configs.json" + + - name: ccache stats + if: always() + run: ccache -s || true + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: no-malloc-logs + path: | + build-*/make-check.log + build-*/test-suite.log + build-*/config.log + if-no-files-found: ignore diff --git a/.github/workflows/opensslcoexist.yml b/.github/workflows/opensslcoexist.yml index 102d7ec07e..e6a55c5c4c 100644 --- a/.github/workflows/opensslcoexist.yml +++ b/.github/workflows/opensslcoexist.yml @@ -14,38 +14,78 @@ concurrency: # END OF COMMON SECTION jobs: + # All former runner-per-config matrix entries build on one runner via + # .github/scripts/parallel-make-check.py (see os-check.yml for the full + # pattern): each config in its own out-of-tree ("VPATH") build directory + # off one checkout/autogen, checks on a pool of one-per-CPU worker + # threads, longest first. bubblewrap gives every test script its own + # network namespace so concurrent checks cannot collide on TCP/UDP ports + # (do not set AM_BWRAPPED here - that would disable it). make_check: - strategy: - matrix: - config: [ - # Add new configs here - '--verbose --enable-all --disable-all-osp --disable-opensslall --enable-opensslcoexist CPPFLAGS="-DNO_WOLFSSL_CIPHER_SUITE_TEST -pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"', - '--verbose --enable-all --disable-all-osp --disable-opensslall --enable-opensslcoexist CPPFLAGS="-DNO_WOLFSSL_CIPHER_SUITE_TEST -pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_OPENSSL_COEXIST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"' - ] name: make check if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} runs-on: ubuntu-24.04 - # This should be a safe limit for the tests to run. - timeout-minutes: 6 + # Generous for a cold ccache; warm reruns finish in a fraction. + timeout-minutes: 15 steps: - uses: actions/checkout@v4 name: Checkout wolfSSL - - name: Test --enable-opensslcoexist and TEST_OPENSSL_COEXIST - run: | - ./autogen.sh || $(exit 2) - ./configure ${{ matrix.config }} || $(exit 3) - make -j 4 || $(exit 4) - make check + - name: Install dependencies + uses: ./.github/actions/install-apt-deps + with: + packages: autoconf automake libtool build-essential bubblewrap - - name: Print errors - if: ${{ failure() }} + # ccache via the cross-platform composite; the script passes the + # compiler to configure as CC="ccache gcc" (or a per-config "cc"). + - name: Set up ccache + uses: ./.github/actions/ccache-setup + with: + workflow-id: opensslcoexist + max-size: 150M + + # Ubuntu 24.04 can restrict unprivileged user namespaces via AppArmor, + # which would stop the test scripts from re-execing under + # bwrap --unshare-net (their port-isolation mechanism). + - name: Allow unprivileged user namespaces (for bwrap) + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true + + # The JSON list below is the former runner-per-config matrix; add new + # configs as new entries (a "comment" key is allowed for notes). + # "minutes" is the expected duration driving longest-first scheduling: + # take it from the Minutes column of a previous run's step summary, or + # omit it for a new config (defaults to 1) and refresh later. The list + # is kept sorted by minutes for readability, but the schedule sorts by + # the values, not list order. + - name: Build and make check all configs (parallel, out-of-tree) run: | - for file in config.log scripts/*.log - do - if [ -f "$file" ]; then - echo "${file}:" - cat "$file" - echo "========================================================================" - fi - done + cat > "$RUNNER_TEMP/opensslcoexist-configs.json" <<'EOF' + [ + {"name": "coexist", "minutes": 2.5, + "configure": ["--verbose", "--enable-all", "--disable-all-osp", + "--disable-opensslall", "--enable-opensslcoexist", + "CPPFLAGS=-DNO_WOLFSSL_CIPHER_SUITE_TEST -pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"]}, + {"name": "test-coexist", "minutes": 2.5, + "configure": ["--verbose", "--enable-all", "--disable-all-osp", + "--disable-opensslall", "--enable-opensslcoexist", + "CPPFLAGS=-DNO_WOLFSSL_CIPHER_SUITE_TEST -pedantic -Wdeclaration-after-statement -Wnull-dereference -DTEST_OPENSSL_COEXIST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"]} + ] + EOF + .github/scripts/parallel-make-check.py \ + --private-dir=certs \ + "$RUNNER_TEMP/opensslcoexist-configs.json" + + - name: ccache stats + if: always() + run: ccache -s || true + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: opensslcoexist-logs + path: | + build-*/make-check.log + build-*/test-suite.log + build-*/config.log + if-no-files-found: ignore diff --git a/.github/workflows/os-check.yml b/.github/workflows/os-check.yml index 1519bd0c88..10aa21d26b 100644 --- a/.github/workflows/os-check.yml +++ b/.github/workflows/os-check.yml @@ -4,9 +4,18 @@ name: Ubuntu-Macos-Windows Tests on: push: branches: [ 'release/**' ] + # Docs-only changes cannot affect the build/test matrix - skip the + # run for them. Keep this list narrow (markdown + doc/ only); + # do not add cert/test data extensions here. + paths-ignore: + - '**/*.md' + - 'doc/**' pull_request: types: [opened, synchronize, reopened, ready_for_review] branches: [ '*' ] + paths-ignore: + - '**/*.md' + - 'doc/**' concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -17,133 +26,35 @@ jobs: # Ubuntu config matrix. macOS is covered separately by make_check_macos # below with a curated subset; configs here either have equivalent macOS # coverage there or exercise no Darwin-specific code. + # + # The config list is built by a small fixed pool of shard runners: every + # shard job runs the generic .github/scripts/parallel-make-check.py on + # the same JSON list below with --shard K/N, which deals the configs + # across the N shards greedily by their "minutes" weight so every shard + # carries a similar load. Within a shard each config builds in its own + # out-of-tree ("VPATH") build directory off one checkout/autogen, and + # the checks run on a pool of one-per-CPU worker threads, longest first; + # per-config times and thread/CPU efficiency land in each shard's step + # summary (same machinery as smoke-test.yml). bubblewrap lets the script + # tests re-exec under bwrap --unshare-net so concurrent checks cannot + # collide on TCP/UDP ports (do not set AM_BWRAPPED here - that would + # disable it). make_check_linux: + name: make check linux strategy: fail-fast: false matrix: - config: [ - # Add new configs here - '', - '--enable-all --enable-asn=template', - '--enable-all --enable-asn=original', - '--enable-all --enable-asn=template CPPFLAGS=-DWOLFSSL_OLD_OID_SUM', - '--enable-all --enable-asn=original CPPFLAGS=-DWOLFSSL_OLD_OID_SUM', - '--enable-harden-tls', - '--enable-tls13 --enable-session-ticket --enable-dtls --enable-dtls13 - --enable-opensslextra --enable-sessioncerts - CPPFLAGS=''-DWOLFSSL_DTLS_NO_HVR_ON_RESUME -DHAVE_EXT_CACHE - -DWOLFSSL_TICKET_HAVE_ID -DHAVE_EX_DATA -DSESSION_CACHE_DYNAMIC_MEM'' ', - '--enable-all --enable-secure-renegotiation', - '--enable-all --enable-haproxy --enable-quic', - '--enable-dtls --enable-dtls13 --enable-earlydata - --enable-session-ticket --enable-psk - CPPFLAGS=''-DWOLFSSL_DTLS13_NO_HRR_ON_RESUME'' ', - '--enable-all --enable-dtls13 --enable-dtls-frag-ch --disable-mlkem', - '--enable-dtls --enable-dtlscid --enable-dtls13 --enable-secure-renegotiation - --enable-psk --enable-aesccm --enable-nullcipher - CPPFLAGS=-DWOLFSSL_STATIC_RSA', - '--enable-she=extended --enable-cryptocb --enable-cryptocbutils - CPPFLAGS=''-DWC_SHE_SW_DEFAULT'' ', - '--enable-she=standard --enable-cmac', - '--enable-she=extended --enable-cmac --enable-cryptocb --enable-cryptocbutils', - '--enable-she=standard --enable-cmac CPPFLAGS=''-DNO_WC_SHE_IMPORT_M123'' ', - '--enable-she=extended --enable-cmac --enable-cryptocb --enable-cryptocbutils - CPPFLAGS=''-DNO_WC_SHE_GETUID -DNO_WC_SHE_GETCOUNTER -DNO_WC_SHE_EXPORTKEY'' ', - '--enable-she=standard --enable-cmac --enable-cryptocb --enable-cryptocbutils - CPPFLAGS=''-DWC_SHE_SW_DEFAULT'' ', - '--enable-all CPPFLAGS=''-DNO_AES_192 -DNO_AES_256'' ', - '--enable-sniffer --enable-curve25519 --enable-curve448 --enable-enckeys - CPPFLAGS=-DWOLFSSL_DH_EXTRA', - '--enable-dtls --enable-dtls13 --enable-dtls-frag-ch - --enable-dtls-mtu CPPFLAGS=-DWOLFSSL_DTLS_RECORDS_CAN_SPAN_DATAGRAMS', - '--enable-opensslall --enable-opensslextra CPPFLAGS=-DWC_RNG_SEED_CB', - '--enable-opensslall --enable-opensslextra - CPPFLAGS=''-DWC_RNG_SEED_CB -DWOLFSSL_NO_GETPID'' ', - '--enable-opensslextra CPPFLAGS=''-DWOLFSSL_NO_CA_NAMES'' ', - '--enable-opensslextra=x509small', - '--disable-sys-ca-certs', - '--enable-all CPPFLAGS=-DWOLFSSL_DEBUG_CERTS ', - '--enable-all CPPFLAGS="-DWOLFSSL_CHECK_MEM_ZERO"', - '--enable-dtls --enable-dtls13 --enable-ocspstapling --enable-ocspstapling2 - --enable-cert-setup-cb --enable-sessioncerts', - '--enable-dtls --enable-dtls13 --enable-tls13 - CPPFLAGS=-DWOLFSSL_TLS13_IGNORE_PT_ALERT_ON_ENC', - '--enable-all --enable-certgencache', - '--enable-all --enable-dilithium --enable-cryptocb --enable-cryptocbutils --enable-pkcallbacks', - '--enable-cryptocb --enable-aesgcm CPPFLAGS="-DWOLF_CRYPTO_CB_AES_SETKEY"', - '--enable-cryptocb --enable-keygen --enable-cryptocbutils=setkey', - '--enable-cryptocb --enable-keygen --enable-cryptocbutils CPPFLAGS="-DWOLF_CRYPTO_CB_AES_SETKEY"', - '--enable-cryptocb --enable-keygen --enable-aesgcm --enable-cryptocbutils=setkey,free CPPFLAGS="-DWOLF_CRYPTO_CB_AES_SETKEY"', - '--enable-cryptocb --enable-keygen --enable-cryptocbutils=export', - '--enable-cryptocb --enable-keygen CPPFLAGS="-DWOLF_CRYPTO_CB_EXPORT_KEY"', - '--enable-cryptocb --enable-keygen --enable-aesgcm --enable-cryptocbutils=setkey,free,export CPPFLAGS="-DWOLF_CRYPTO_CB_AES_SETKEY"', - '--enable-cryptocb --enable-keygen --enable-cryptocbutils=setkey,export CPPFLAGS="-DWOLF_CRYPTO_CB_FIND"', - 'CPPFLAGS=-DWOLFSSL_NO_CLIENT_AUTH', - 'CPPFLAGS=''-DNO_WOLFSSL_CLIENT -DWOLFSSL_NO_CLIENT_AUTH''', - 'CPPFLAGS=''-DNO_WOLFSSL_SERVER -DWOLFSSL_NO_CLIENT_AUTH''', - '--enable-all CPPFLAGS=-DNO_WOLFSSL_CLIENT', - '--enable-all CPPFLAGS=-DNO_WOLFSSL_SERVER', - '--enable-all CPPFLAGS=-DWOLFSSL_NO_CLIENT_AUTH', - '--enable-all CPPFLAGS=''-DNO_WOLFSSL_CLIENT -DWOLFSSL_NO_CLIENT_AUTH''', - '--enable-all CPPFLAGS=''-DNO_WOLFSSL_SERVER -DWOLFSSL_NO_CLIENT_AUTH''', - '--enable-ocsp --enable-ocsp-responder --enable-ocspstapling CPPFLAGS="-DWOLFSSL_NONBLOCK_OCSP" --enable-maxfragment', - '--enable-all CPPFLAGS=-DWOLFSSL_HASH_KEEP', - '--enable-all --enable-writedup', - '--enable-ascon --enable-experimental', - '--enable-ascon CPPFLAGS=-DWOLFSSL_ASCON_UNROLL --enable-experimental', - # PKCS#7 with RSA-PSS (CMS RSASSA-PSS signers) - '--enable-pkcs7 CPPFLAGS=-DWC_RSA_PSS', - # PKCS#7 without RSA-PSS - '--enable-pkcs7', - 'CPPFLAGS=''-DWOLFSSL_EXTRA'' ', - '--enable-coding=no', - '--disable-sni --disable-ecc --disable-tls13 --disable-secure-renegotiation-info', - 'CPPFLAGS=-DWOLFSSL_BLIND_PRIVATE_KEY', - '--enable-sessionexport --enable-dtls --enable-dtls13', - '--enable-cryptocb --enable-aesgcm CPPFLAGS="-DWOLF_CRYPTO_CB_AES_SETKEY -DWOLF_CRYPTO_CB_FREE"', - '--disable-tls --enable-cryptocb --enable-aesgcm CPPFLAGS="-DWOLF_CRYPTO_CB_AES_SETKEY -DWOLF_CRYPTO_CB_FREE"', - '--enable-cryptocb --enable-keygen CPPFLAGS="-DWOLF_CRYPTO_CB_FIND"', - '--disable-examples CPPFLAGS=-DWOLFSSL_NO_MALLOC', - 'CPPFLAGS=-DNO_WOLFSSL_CLIENT', - 'CPPFLAGS=-DNO_WOLFSSL_SERVER', - '--enable-lms=small,verify-only --enable-xmss=small,verify-only', - '--enable-opensslall --enable-ecc CPPFLAGS="-DWC_ALLOW_ECC_ZERO_HASH"', - # Non-blocking ECC + Curve25519 + RSA + DH on the default SP word - # size for the host (sp_c64.c on x86_64). RSA/DH non-block require - # RSA_LOW_MEM (CRT path is not supported in non-block mode). - '--enable-curve25519=nonblock --enable-ecc=nonblock --enable-rsa=nonblock --enable-dh=nonblock --enable-sp=yes,nonblock CPPFLAGS="-DWOLFSSL_PUBLIC_MP -DWOLFSSL_DEBUG_NONBLOCK -DRSA_LOW_MEM"', - # Same configuration but force SP_WORD_SIZE=32 to exercise sp_c32.c - # on a 64-bit host. The two builds together cover both generated - # variants of mod_exp__nb / RSA / DH wrappers. - '--enable-curve25519=nonblock --enable-ecc=nonblock --enable-rsa=nonblock --enable-dh=nonblock --enable-sp=yes,nonblock CPPFLAGS="-DWOLFSSL_PUBLIC_MP -DWOLFSSL_DEBUG_NONBLOCK -DRSA_LOW_MEM -DSP_WORD_SIZE=32"', - '--enable-certreq --enable-certext --enable-certgen --disable-secure-renegotiation-info CPPFLAGS="-DNO_TLS"', - # Minimal DTLS 1.3 client-only build with the SHA-224/384/512/3 - # hash families disabled. SHA-256 (used by TLS_AES_128_GCM_SHA256) - # and SHA-1 remain enabled. - '--enable-dtls13 --disable-tlsv12 --disable-oldtls --disable-rsa --disable-dh - --disable-aescbc --disable-aesecb --disable-md5 --disable-chacha - --disable-poly1305 --disable-errorstrings --disable-asn-print - --disable-eccshamir --disable-base64encode --disable-coding --disable-sni - --disable-sha224 --disable-sha384 --disable-sha512 --disable-sha3 - --enable-aesgcm=small --enable-sp-math --enable-sp=smallec256 --disable-sp-asm - CPPFLAGS=''-DNO_WOLFSSL_SERVER -DWOLFSSL_NO_TLS12 -DNO_SESSION_CACHE - -DWOLFSSL_AES_NO_UNROLL -DUSE_SLOW_SHA256 -DWOLFSSL_NO_ASYNC_IO - -DWOLFSSL_DTLS_ONLY'' ', - '--enable-opensslextra --disable-filesystem CPPFLAGS="-DNO_BIO"', - 'CPPFLAGS=-DNO_VERIFY_OID', - 'CPPFLAGS="-DNO_VERIFY_OID -DWOLFSSL_FPKI"', - ] - name: make check linux + shard: [1, 2, 3, 4] if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} runs-on: ubuntu-24.04 - # This should be a safe limit for the tests to run. - timeout-minutes: 14 + # The full set measures ~272 thread-minutes cold (~90 warm), i.e. about + # 68 thread-minutes per shard / 4 worker threads: ~20 min of wall per + # shard cold and well under 10 warm, plus ~2-3 min of + # checkout/deps/autogen overhead. + timeout-minutes: 30 + env: + CCACHE_MAXSIZE: 500M steps: - # Local composite actions (./.github/actions/*) need the repo on - # disk before the runner can resolve them. The autotools-project - # step further down does its own checkout into the workspace, so - # this explicit checkout is only required for the ccache-setup - # composite below. - uses: actions/checkout@v4 # tlslite-ng is consumed by scripts/multi-msg-record.test (run from @@ -153,62 +64,357 @@ jobs: python-version: '3.x' - run: pip install tlslite-ng - # ccache cuts ~50% off rebuild time. /usr/lib/ccache is prepended to - # PATH so gcc/cc invocations from the autotools action are - # transparently intercepted - no other step needs to change. - - name: Set up ccache - uses: ./.github/actions/ccache-setup + - name: Install dependencies + uses: ./.github/actions/install-apt-deps with: - workflow-id: os-check-linux + packages: autoconf automake libtool build-essential bubblewrap ccache - - name: Build and test wolfSSL - uses: wolfSSL/actions-build-autotools-project@v1 + # Ubuntu 24.04 can restrict unprivileged user namespaces via AppArmor, + # which would stop the test scripts from re-execing under + # bwrap --unshare-net (their port-isolation mechanism). + - name: Allow unprivileged user namespaces (for bwrap) + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true + + - name: Restore ccache + uses: actions/cache@v4 with: - configure: CFLAGS="-pedantic -Wdeclaration-after-statement -Wnull-dereference -Wno-overlength-strings -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE" ${{ matrix.config }} - check: true + path: ~/.cache/ccache + # Per-shard cache lineage: each shard compiles a distinct config + # subset. Re-balancing "minutes" can move configs between shards; + # that costs one rebuild of whatever moved. + key: os-check-linux-ccache-${{ matrix.shard }}-${{ github.base_ref || github.ref_name }}-${{ github.sha }} + restore-keys: | + os-check-linux-ccache-${{ matrix.shard }}-${{ github.base_ref || github.ref_name }}- + os-check-linux-ccache-${{ matrix.shard }}- + os-check-linux-ccache- - - name: ccache stats (post-build) + - name: autogen + run: | + ccache -z + ./autogen.sh + + # The JSON list below is the former runner-per-config matrix; add new + # configs here as new entries (a "comment" key is allowed for notes). + # "minutes" is the expected duration driving longest-first scheduling + # and shard balancing: take it from the Minutes column of a previous + # run's step summary, or omit it for a new config (defaults to 1) and + # refresh later - a stale value only packs the schedule worse. The + # list is kept sorted by minutes for readability, but the schedule + # sorts by the values, not list order. + # The CFLAGS that were previously passed to configure are applied at + # make time via --cflags, unchanged. --private-dir=certs gives every + # build dir its own certs/ copy: crl-gen-openssl.test writes generated + # CRLs under certs/crl/, which would race through the shared VPATH + # certs symlink. + - name: Build and make check this shard's configs (parallel, out-of-tree) + run: | + cat > "$RUNNER_TEMP/os-check-configs.json" <<'EOF' + [ + {"name": "all-no-client-auth", "minutes": 9.0, + "configure": ["--enable-all", "CPPFLAGS=-DWOLFSSL_NO_CLIENT_AUTH"]}, + {"name": "all-dilithium-cryptocb", "minutes": 8.9, + "configure": ["--enable-all", "--enable-dilithium", "--enable-cryptocb", + "--enable-cryptocbutils", "--enable-pkcallbacks"]}, + {"name": "all-haproxy-quic", "minutes": 8.6, + "configure": ["--enable-all", "--enable-haproxy", "--enable-quic"]}, + {"name": "all-asn-template", "minutes": 8.5, + "configure": ["--enable-all", "--enable-asn=template"]}, + {"name": "all-asn-template-old-oid-sum", "minutes": 8.5, + "configure": ["--enable-all", "--enable-asn=template", "CPPFLAGS=-DWOLFSSL_OLD_OID_SUM"]}, + {"name": "all-asn-original-old-oid-sum", "minutes": 8.4, + "configure": ["--enable-all", "--enable-asn=original", "CPPFLAGS=-DWOLFSSL_OLD_OID_SUM"]}, + {"name": "all-asn-original", "minutes": 8.3, + "configure": ["--enable-all", "--enable-asn=original"]}, + {"name": "all-certgencache", "minutes": 8.3, + "configure": ["--enable-all", "--enable-certgencache"]}, + {"name": "all-dtls13-frag-ch-no-mlkem", "minutes": 8.2, + "configure": ["--enable-all", "--enable-dtls13", "--enable-dtls-frag-ch", + "--disable-mlkem"]}, + {"name": "all-check-mem-zero", "minutes": 7.9, + "configure": ["--enable-all", "CPPFLAGS=-DWOLFSSL_CHECK_MEM_ZERO"]}, + {"name": "all-secure-renegotiation", "minutes": 7.8, + "configure": ["--enable-all", "--enable-secure-renegotiation"]}, + {"name": "all-debug-certs", "minutes": 7.8, + "configure": ["--enable-all", "CPPFLAGS=-DWOLFSSL_DEBUG_CERTS"]}, + {"name": "all-hash-keep", "minutes": 7.8, + "configure": ["--enable-all", "CPPFLAGS=-DWOLFSSL_HASH_KEEP"]}, + {"name": "all-no-aes-192-256", "minutes": 7.5, + "configure": ["--enable-all", "CPPFLAGS=-DNO_AES_192 -DNO_AES_256"]}, + {"name": "all-writedup", "minutes": 6.9, + "configure": ["--enable-all", "--enable-writedup"]}, + {"name": "all-no-server", "minutes": 5.0, + "configure": ["--enable-all", "CPPFLAGS=-DNO_WOLFSSL_SERVER"]}, + {"name": "nonblock-sp-c32", "minutes": 5.0, + "comment": "Same but forcing SP_WORD_SIZE=32 to exercise sp_c32.c on a 64-bit host; the two builds together cover both generated variants of mod_exp__nb / RSA / DH wrappers.", + "configure": ["--enable-curve25519=nonblock", "--enable-ecc=nonblock", + "--enable-rsa=nonblock", "--enable-dh=nonblock", "--enable-sp=yes,nonblock", + "CPPFLAGS=-DWOLFSSL_PUBLIC_MP -DWOLFSSL_DEBUG_NONBLOCK -DRSA_LOW_MEM -DSP_WORD_SIZE=32"]}, + {"name": "all-no-server-no-client-auth", "minutes": 4.8, + "configure": ["--enable-all", "CPPFLAGS=-DNO_WOLFSSL_SERVER -DWOLFSSL_NO_CLIENT_AUTH"]}, + {"name": "all-no-client-no-client-auth", "minutes": 4.4, + "configure": ["--enable-all", "CPPFLAGS=-DNO_WOLFSSL_CLIENT -DWOLFSSL_NO_CLIENT_AUTH"]}, + {"name": "all-no-client", "minutes": 4.2, + "configure": ["--enable-all", "CPPFLAGS=-DNO_WOLFSSL_CLIENT"]}, + {"name": "nonblock-sp-c64", "minutes": 4.2, + "comment": "Non-blocking ECC/Curve25519/RSA/DH on the host default SP word size (sp_c64.c on x86_64); RSA/DH non-block require RSA_LOW_MEM (the CRT path is not supported in non-block mode).", + "configure": ["--enable-curve25519=nonblock", "--enable-ecc=nonblock", + "--enable-rsa=nonblock", "--enable-dh=nonblock", "--enable-sp=yes,nonblock", + "CPPFLAGS=-DWOLFSSL_PUBLIC_MP -DWOLFSSL_DEBUG_NONBLOCK -DRSA_LOW_MEM"]}, + {"name": "tls13-dtls13-session-misc", "minutes": 3.3, + "configure": ["--enable-tls13", "--enable-session-ticket", "--enable-dtls", + "--enable-dtls13", "--enable-opensslextra", "--enable-sessioncerts", + "CPPFLAGS=-DWOLFSSL_DTLS_NO_HVR_ON_RESUME -DHAVE_EXT_CACHE -DWOLFSSL_TICKET_HAVE_ID -DHAVE_EX_DATA -DSESSION_CACHE_DYNAMIC_MEM"]}, + {"name": "dtls-cid-renego-psk", "minutes": 3.3, + "configure": ["--enable-dtls", "--enable-dtlscid", "--enable-dtls13", + "--enable-secure-renegotiation", "--enable-psk", "--enable-aesccm", + "--enable-nullcipher", "CPPFLAGS=-DWOLFSSL_STATIC_RSA"]}, + {"name": "dtls13-ocspstapling-cert-cb", "minutes": 3.1, + "configure": ["--enable-dtls", "--enable-dtls13", "--enable-ocspstapling", + "--enable-ocspstapling2", "--enable-cert-setup-cb", "--enable-sessioncerts"]}, + {"name": "user-settings-all-compat", "minutes": 3.0, + "comment": "user_settings_all.h with the compatibility layer enabled by flipping its \"#if 0\" block, as a build-dir copy.", + "user_settings": "examples/configs/user_settings_all.h", + "cflags": "", + "configure": ["--enable-usersettings"], "prepare": [["sed", "-i", "s/if 0/if 1/", "user_settings.h"]]}, + {"name": "dtls13-earlydata-psk-no-hrr", "minutes": 2.9, + "configure": ["--enable-dtls", "--enable-dtls13", "--enable-earlydata", + "--enable-session-ticket", "--enable-psk", "CPPFLAGS=-DWOLFSSL_DTLS13_NO_HRR_ON_RESUME"]}, + {"name": "ocsp-responder-nonblock-maxfrag", "minutes": 2.8, + "configure": ["--enable-ocsp", "--enable-ocsp-responder", "--enable-ocspstapling", + "CPPFLAGS=-DWOLFSSL_NONBLOCK_OCSP", "--enable-maxfragment"]}, + {"name": "dtls-records-span-datagrams", "minutes": 2.7, + "configure": ["--enable-dtls", "--enable-dtls13", "--enable-dtls-frag-ch", + "--enable-dtls-mtu", "CPPFLAGS=-DWOLFSSL_DTLS_RECORDS_CAN_SPAN_DATAGRAMS"]}, + {"name": "opensslextra-no-ca-names", "minutes": 2.6, + "configure": ["--enable-opensslextra", "CPPFLAGS=-DWOLFSSL_NO_CA_NAMES"]}, + {"name": "sessionexport-dtls13", "minutes": 2.6, + "configure": ["--enable-sessionexport", "--enable-dtls", "--enable-dtls13"]}, + {"name": "lms-xmss-verify-only", "minutes": 2.5, + "configure": ["--enable-lms=small,verify-only", "--enable-xmss=small,verify-only"]}, + {"name": "opensslall-rng-seed-cb", "minutes": 2.2, + "configure": ["--enable-opensslall", "--enable-opensslextra", "CPPFLAGS=-DWC_RNG_SEED_CB"]}, + {"name": "opensslall-ecc-zero-hash", "minutes": 2.2, + "configure": ["--enable-opensslall", "--enable-ecc", "CPPFLAGS=-DWC_ALLOW_ECC_ZERO_HASH"]}, + {"name": "she-ext-cmac-no-she-misc", "minutes": 2.2, + "configure": ["--enable-she=extended", "--enable-cmac", "--enable-cryptocb", + "--enable-cryptocbutils", + "CPPFLAGS=-DNO_WC_SHE_GETUID -DNO_WC_SHE_GETCOUNTER -DNO_WC_SHE_EXPORTKEY"]}, + {"name": "she-std-cmac-cryptocb-sw-default", "minutes": 2.2, + "configure": ["--enable-she=standard", "--enable-cmac", "--enable-cryptocb", + "--enable-cryptocbutils", "CPPFLAGS=-DWC_SHE_SW_DEFAULT"]}, + {"name": "sniffer-curves-enckeys", "minutes": 2.2, + "configure": ["--enable-sniffer", "--enable-curve25519", "--enable-curve448", + "--enable-enckeys", "CPPFLAGS=-DWOLFSSL_DH_EXTRA"]}, + {"name": "cryptocb-keygen-utils-aes-setkey", "minutes": 2.2, + "configure": ["--enable-cryptocb", "--enable-keygen", "--enable-cryptocbutils", + "CPPFLAGS=-DWOLF_CRYPTO_CB_AES_SETKEY"]}, + {"name": "cryptocb-utils-setkey-free", "minutes": 2.2, + "configure": ["--enable-cryptocb", "--enable-keygen", "--enable-aesgcm", + "--enable-cryptocbutils=setkey,free", "CPPFLAGS=-DWOLF_CRYPTO_CB_AES_SETKEY"]}, + {"name": "cryptocb-keygen-utils-export", "minutes": 2.2, + "configure": ["--enable-cryptocb", "--enable-keygen", "--enable-cryptocbutils=export"]}, + {"name": "cryptocb-keygen-export-key", "minutes": 2.2, + "configure": ["--enable-cryptocb", "--enable-keygen", + "CPPFLAGS=-DWOLF_CRYPTO_CB_EXPORT_KEY"]}, + {"name": "cryptocb-utils-setkey-export-find", "minutes": 2.2, + "configure": ["--enable-cryptocb", "--enable-keygen", + "--enable-cryptocbutils=setkey,export", "CPPFLAGS=-DWOLF_CRYPTO_CB_FIND"]}, + {"name": "opensslall-rng-seed-cb-no-getpid", "minutes": 2.1, + "configure": ["--enable-opensslall", "--enable-opensslextra", + "CPPFLAGS=-DWC_RNG_SEED_CB -DWOLFSSL_NO_GETPID"]}, + {"name": "dtls13-ignore-pt-alert", "minutes": 2.1, + "configure": ["--enable-dtls", "--enable-dtls13", "--enable-tls13", + "CPPFLAGS=-DWOLFSSL_TLS13_IGNORE_PT_ALERT_ON_ENC"]}, + {"name": "cryptocb-utils-setkey-free-export", "minutes": 2.1, + "configure": ["--enable-cryptocb", "--enable-keygen", "--enable-aesgcm", + "--enable-cryptocbutils=setkey,free,export", "CPPFLAGS=-DWOLF_CRYPTO_CB_AES_SETKEY"]}, + {"name": "cryptocb-aesgcm-setkey-free", "minutes": 2.1, + "configure": ["--enable-cryptocb", "--enable-aesgcm", + "CPPFLAGS=-DWOLF_CRYPTO_CB_AES_SETKEY -DWOLF_CRYPTO_CB_FREE"]}, + {"name": "opensslextra-x509small", "minutes": 2.0, + "configure": ["--enable-opensslextra=x509small"]}, + {"name": "cryptocb-keygen-find", "minutes": 2.0, + "configure": ["--enable-cryptocb", "--enable-keygen", "CPPFLAGS=-DWOLF_CRYPTO_CB_FIND"]}, + {"name": "user-settings-all", "minutes": 2.0, + "comment": "The user_settings.h header-driven build path is distinct from the autotools-driven --enable-all path; full make check.", + "user_settings": "examples/configs/user_settings_all.h", + "cflags": "", + "configure": ["--enable-usersettings"]}, + {"name": "she-ext-cryptocb-sw-default", "minutes": 1.9, + "configure": ["--enable-she=extended", "--enable-cryptocb", "--enable-cryptocbutils", + "CPPFLAGS=-DWC_SHE_SW_DEFAULT"]}, + {"name": "cryptocb-aesgcm-aes-setkey", "minutes": 1.9, + "configure": ["--enable-cryptocb", "--enable-aesgcm", + "CPPFLAGS=-DWOLF_CRYPTO_CB_AES_SETKEY"]}, + {"name": "cryptocb-keygen-utils-setkey", "minutes": 1.9, + "configure": ["--enable-cryptocb", "--enable-keygen", "--enable-cryptocbutils=setkey"]}, + {"name": "pkcs7-rsa-pss", "minutes": 1.9, + "comment": "PKCS#7 with RSA-PSS (CMS RSASSA-PSS signers)", + "configure": ["--enable-pkcs7", "CPPFLAGS=-DWC_RSA_PSS"]}, + {"name": "blind-private-key", "minutes": 1.9, + "configure": ["CPPFLAGS=-DWOLFSSL_BLIND_PRIVATE_KEY"]}, + {"name": "certgen-no-tls", "minutes": 1.9, + "configure": ["--enable-certreq", "--enable-certext", "--enable-certgen", + "--disable-secure-renegotiation-info", "CPPFLAGS=-DNO_TLS"]}, + {"name": "no-sys-ca-certs", "minutes": 1.8, "configure": ["--disable-sys-ca-certs"]}, + {"name": "no-client-auth", "minutes": 1.8, + "configure": ["CPPFLAGS=-DWOLFSSL_NO_CLIENT_AUTH"]}, + {"name": "harden-tls", "minutes": 1.7, "configure": ["--enable-harden-tls"]}, + {"name": "no-sni-ecc-tls13-scr-info", "minutes": 1.7, + "configure": ["--disable-sni", "--disable-ecc", "--disable-tls13", + "--disable-secure-renegotiation-info"]}, + {"name": "default", "minutes": 1.6}, + {"name": "no-client-no-client-auth", "minutes": 1.6, + "configure": ["CPPFLAGS=-DNO_WOLFSSL_CLIENT -DWOLFSSL_NO_CLIENT_AUTH"]}, + {"name": "ascon-experimental", "minutes": 1.6, + "configure": ["--enable-ascon", "--enable-experimental"]}, + {"name": "ascon-unroll-experimental", "minutes": 1.6, + "configure": ["--enable-ascon", "CPPFLAGS=-DWOLFSSL_ASCON_UNROLL", + "--enable-experimental"]}, + {"name": "wolfssl-extra", "minutes": 1.6, "configure": ["CPPFLAGS=-DWOLFSSL_EXTRA"]}, + {"name": "coding-no", "minutes": 1.5, "configure": ["--enable-coding=no"]}, + {"name": "she-ext-cmac-cryptocb", "minutes": 1.3, + "configure": ["--enable-she=extended", "--enable-cmac", "--enable-cryptocb", + "--enable-cryptocbutils"]}, + {"name": "she-std-cmac-no-import-m123", "minutes": 1.3, + "configure": ["--enable-she=standard", "--enable-cmac", + "CPPFLAGS=-DNO_WC_SHE_IMPORT_M123"]}, + {"name": "pkcs7", "minutes": 1.3, + "comment": "PKCS#7 without RSA-PSS", + "configure": ["--enable-pkcs7"]}, + {"name": "no-tls-cryptocb-aesgcm-setkey-free", "minutes": 1.3, + "configure": ["--disable-tls", "--enable-cryptocb", "--enable-aesgcm", + "CPPFLAGS=-DWOLF_CRYPTO_CB_AES_SETKEY -DWOLF_CRYPTO_CB_FREE"]}, + {"name": "she-std-cmac", "minutes": 1.2, + "configure": ["--enable-she=standard", "--enable-cmac"]}, + {"name": "no-verify-oid-fpki", "minutes": 1.2, + "configure": ["CPPFLAGS=-DNO_VERIFY_OID -DWOLFSSL_FPKI"]}, + {"name": "no-verify-oid", "minutes": 1.1, "configure": ["CPPFLAGS=-DNO_VERIFY_OID"]}, + {"name": "no-server-no-client-auth", "minutes": 1.0, + "configure": ["CPPFLAGS=-DNO_WOLFSSL_SERVER -DWOLFSSL_NO_CLIENT_AUTH"]}, + {"name": "no-wolfssl-client", "minutes": 1.0, + "configure": ["CPPFLAGS=-DNO_WOLFSSL_CLIENT"]}, + {"name": "testwolfcrypt-ca", "minutes": 1.0, + "comment": "user_settings.h builds running only testwolfcrypt: pure crypto, no platform-specific features, so Linux-only coverage is sufficient. Not converted: user_settings_pq.h (requires --enable-experimental) and user_settings_baremetal.h (static memory, custom platform).", + "user_settings": "examples/configs/user_settings_ca.h", + "cflags": "", + "configure": ["--enable-usersettings", "--disable-examples"], "check": false, "run": [["wolfcrypt/test/testwolfcrypt"]]}, + {"name": "testwolfcrypt-dtls13", "minutes": 1.0, + "user_settings": "examples/configs/user_settings_dtls13.h", + "cflags": "", + "configure": ["--enable-usersettings", "--disable-examples"], "check": false, "run": [["wolfcrypt/test/testwolfcrypt"]]}, + {"name": "testwolfcrypt-ebsnet", "minutes": 1.0, + "user_settings": "examples/configs/user_settings_EBSnet.h", + "cflags": "", + "configure": ["--enable-usersettings", "--disable-examples"], "check": false, "run": [["wolfcrypt/test/testwolfcrypt"]]}, + {"name": "testwolfcrypt-eccnonblock", "minutes": 1.0, + "user_settings": "examples/configs/user_settings_eccnonblock.h", + "cflags": "", + "configure": ["--enable-usersettings", "--disable-examples"], "check": false, "run": [["wolfcrypt/test/testwolfcrypt"]]}, + {"name": "testwolfcrypt-curve25519nonblock", "minutes": 1.0, + "user_settings": "examples/configs/user_settings_curve25519nonblock.h", + "cflags": "", + "configure": ["--enable-usersettings", "--disable-examples"], "check": false, "run": [["wolfcrypt/test/testwolfcrypt"]]}, + {"name": "testwolfcrypt-min-ecc", "minutes": 1.0, + "user_settings": "examples/configs/user_settings_min_ecc.h", + "cflags": "", + "configure": ["--enable-usersettings", "--disable-examples"], "check": false, "run": [["wolfcrypt/test/testwolfcrypt"]]}, + {"name": "testwolfcrypt-openssl-compat", "minutes": 1.0, + "user_settings": "examples/configs/user_settings_openssl_compat.h", + "cflags": "", + "configure": ["--enable-usersettings", "--disable-examples"], "check": false, "run": [["wolfcrypt/test/testwolfcrypt"]]}, + {"name": "testwolfcrypt-pkcs7", "minutes": 1.0, + "user_settings": "examples/configs/user_settings_pkcs7.h", + "cflags": "", + "configure": ["--enable-usersettings", "--disable-examples"], "check": false, "run": [["wolfcrypt/test/testwolfcrypt"]]}, + {"name": "testwolfcrypt-rsa-only", "minutes": 1.0, + "user_settings": "examples/configs/user_settings_rsa_only.h", + "cflags": "", + "configure": ["--enable-usersettings", "--disable-examples"], "check": false, "run": [["wolfcrypt/test/testwolfcrypt"]]}, + {"name": "testwolfcrypt-template", "minutes": 1.0, + "user_settings": "examples/configs/user_settings_template.h", + "cflags": "", + "configure": ["--enable-usersettings", "--disable-examples"], "check": false, "run": [["wolfcrypt/test/testwolfcrypt"]]}, + {"name": "testwolfcrypt-tls12", "minutes": 1.0, + "user_settings": "examples/configs/user_settings_tls12.h", + "cflags": "", + "configure": ["--enable-usersettings", "--disable-examples"], "check": false, "run": [["wolfcrypt/test/testwolfcrypt"]]}, + {"name": "testwolfcrypt-tls13", "minutes": 1.0, + "user_settings": "examples/configs/user_settings_tls13.h", + "cflags": "", + "configure": ["--enable-usersettings", "--disable-examples"], "check": false, "run": [["wolfcrypt/test/testwolfcrypt"]]}, + {"name": "testwolfcrypt-wolfboot-keytools", "minutes": 1.0, + "user_settings": "examples/configs/user_settings_wolfboot_keytools.h", + "cflags": "", + "configure": ["--enable-usersettings", "--disable-examples"], "check": false, "run": [["wolfcrypt/test/testwolfcrypt"]]}, + {"name": "testwolfcrypt-wolfssh", "minutes": 1.0, + "user_settings": "examples/configs/user_settings_wolfssh.h", + "cflags": "", + "configure": ["--enable-usersettings", "--disable-examples"], "check": false, "run": [["wolfcrypt/test/testwolfcrypt"]]}, + {"name": "testwolfcrypt-wolftpm", "minutes": 1.0, + "user_settings": "examples/configs/user_settings_wolftpm.h", + "cflags": "", + "configure": ["--enable-usersettings", "--disable-examples"], "check": false, "run": [["wolfcrypt/test/testwolfcrypt"]]}, + {"name": "no-wolfssl-server", "minutes": 0.9, + "configure": ["CPPFLAGS=-DNO_WOLFSSL_SERVER"]}, + {"name": "dtls13-client-minimal", "minutes": 0.9, + "comment": "Minimal DTLS 1.3 client-only build with the SHA-224/384/512/3 hash families disabled. SHA-256 (used by TLS_AES_128_GCM_SHA256) and SHA-1 remain enabled.", + "configure": ["--enable-dtls13", "--disable-tlsv12", "--disable-oldtls", "--disable-rsa", + "--disable-dh", "--disable-aescbc", "--disable-aesecb", "--disable-md5", + "--disable-chacha", "--disable-poly1305", "--disable-errorstrings", + "--disable-asn-print", "--disable-eccshamir", "--disable-base64encode", + "--disable-coding", "--disable-sni", "--disable-sha224", "--disable-sha384", + "--disable-sha512", "--disable-sha3", "--enable-aesgcm=small", "--enable-sp-math", + "--enable-sp=smallec256", "--disable-sp-asm", + "CPPFLAGS=-DNO_WOLFSSL_SERVER -DWOLFSSL_NO_TLS12 -DNO_SESSION_CACHE -DWOLFSSL_AES_NO_UNROLL -DUSE_SLOW_SHA256 -DWOLFSSL_NO_ASYNC_IO -DWOLFSSL_DTLS_ONLY"]}, + {"name": "opensslextra-no-filesystem-no-bio", "minutes": 0.9, + "configure": ["--enable-opensslextra", "--disable-filesystem", "CPPFLAGS=-DNO_BIO"]}, + {"name": "no-examples-no-malloc", "minutes": 0.8, + "configure": ["--disable-examples", "CPPFLAGS=-DWOLFSSL_NO_MALLOC"]} + ] + EOF + .github/scripts/parallel-make-check.py \ + --shard "${{ matrix.shard }}/${{ strategy.job-total }}" \ + --cflags='-pedantic -Wdeclaration-after-statement -Wnull-dereference -Wno-overlength-strings -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE' \ + --private-dir=certs "$RUNNER_TEMP/os-check-configs.json" + + - name: ccache stats if: always() - run: command -v ccache >/dev/null && ccache -s || echo "ccache not installed - composite likely skipped" + run: ccache -s || true + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: os-check-linux-logs-${{ matrix.shard }} + path: | + build-*/make-check.log + build-*/test-suite.log + build-*/config.log + if-no-files-found: ignore # Curated macOS subset. Each config exists for a Darwin-specific reason; # do not add entries that only re-test platform-agnostic crypto already # covered by the corresponding Linux run. + # + # All configs build on ONE macos runner via + # .github/scripts/parallel-make-check.py, each in its own out-of-tree + # ("VPATH") build directory. Unlike Linux, the checks run one config at + # a time (--threads 1): macOS has no bubblewrap, so concurrent make + # checks would race on TCP/UDP ports. The user_settings_all.h entry is + # the former macOS half of the make_user_settings job, and the + # apple-native-cert-validation entry is the former standalone + # macos-apple-native-cert-validation.yml workflow. make_check_macos: - strategy: - fail-fast: false - matrix: - config: [ - # Default build: --enable-sys-ca-certs is auto-on on macOS, so - # this exercises Apple keychain / system trust loading in - # src/ssl_load.c that has no Linux equivalent. - '', - # Broad key-crypto + Security.framework + opensslextra in one run - # (RSA, ECC, AES, SHA-2/3, ChaCha20-Poly1305, Curve25519/448, HMAC, - # sniffer, DTLS, OCSP, ...). Note: --enable-all does NOT enable - # cryptocb or SHE, so those have their own entries below. - '--enable-all --enable-asn=template', - # Validates the configure-time auto-enable override and that the - # build compiles out the Security.framework code path cleanly -- - # macOS is the only OS where sys-ca-certs is auto-on by default. - '--disable-sys-ca-certs', - # DTLS over BSD sockets on Darwin: connection-ID, fragmented - # ClientHello, secure renegotiation, PSK, AES-CCM, null cipher -- - # exercises recvmsg/MTU/datagram handling that differs from Linux. - '--enable-dtls --enable-dtlscid --enable-dtls13 --enable-secure-renegotiation - --enable-psk --enable-aesccm --enable-nullcipher - CPPFLAGS=-DWOLFSSL_STATIC_RSA', - # Crypto-callback dispatcher under Apple clang. Not covered by - # --enable-all; verifies the cryptocb find/setkey/keygen path - # compiles and runs on the macOS toolchain. - '--enable-cryptocb --enable-keygen --enable-cryptocbutils=setkey', - ] name: make check macos if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} runs-on: macos-latest - # This should be a safe limit for the tests to run. - timeout-minutes: 14 + # Serial checks: roughly the sum of the per-config minutes plus + # one-time setup, with headroom for a cold ccache. + timeout-minutes: 45 steps: + - uses: actions/checkout@v4 + # tlslite-ng is consumed by scripts/multi-msg-record.test (run from # `make check`); without it that test is SKIPped. - uses: actions/setup-python@v5 @@ -216,115 +422,74 @@ jobs: python-version: '3.x' - run: pip install tlslite-ng - - name: Build and test wolfSSL - uses: wolfSSL/actions-build-autotools-project@v1 + # The macos runner images ship without autotools (the old per-config + # matrix got them via wolfSSL/actions-build-autotools-project). + - name: Install autotools (brew) + run: brew install autoconf automake libtool + + # ccache via the cross-platform composite (brew install; compiler + # symlinks on PATH). The script is told --cc= so it does not also + # prefix the compiler with "ccache" - the PATH masquerade already + # intercepts cc/clang. + - name: Set up ccache + uses: ./.github/actions/ccache-setup with: - configure: CFLAGS="-pedantic -Wdeclaration-after-statement -Wnull-dereference -Wno-overlength-strings -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE" ${{ matrix.config }} - check: true + workflow-id: os-check-macos + max-size: 500M - # Run on both OSes: the user_settings.h header-driven build path is - # distinct from the autotools-driven --enable-all path in - # make_check_linux / make_check_macos, and macOS-specific guard ordering - # (e.g. WOLFSSL_SYS_CA_CERTS pulling in Security.framework) needs to be - # exercised under Apple clang here. - make_user_settings: - strategy: - fail-fast: false - matrix: - os: [ ubuntu-24.04, macos-latest ] - user-settings: [ - # Add new user_settings.h here - 'examples/configs/user_settings_all.h', - ] - name: make user_setting.h - if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} - runs-on: ${{ matrix.os }} - # This should be a safe limit for the tests to run. - timeout-minutes: 14 - steps: - # tlslite-ng is consumed by scripts/multi-msg-record.test (run from - # `make check`); without it that test is SKIPped. - - uses: actions/setup-python@v5 + # Same JSON config format as make_check_linux above; "minutes" only + # orders the serial schedule here (longest first). + - name: Build and make check all configs (serial checks, out-of-tree) + run: | + cat > "$RUNNER_TEMP/os-check-macos-configs.json" <<'EOF' + [ + {"name": "all-asn-template", "minutes": 3.0, + "comment": "Broad key-crypto + Security.framework + opensslextra in one run (RSA, ECC, AES, SHA-2/3, ChaCha20-Poly1305, Curve25519/448, HMAC, sniffer, DTLS, OCSP, ...). Note: --enable-all does NOT enable cryptocb or SHE, so those have their own entries.", + "configure": ["--enable-all", "--enable-asn=template"]}, + {"name": "dtls-cid-renego-psk", "minutes": 1.5, + "comment": "DTLS over BSD sockets on Darwin: connection-ID, fragmented ClientHello, secure renegotiation, PSK, AES-CCM, null cipher - exercises recvmsg/MTU/datagram handling that differs from Linux.", + "configure": ["--enable-dtls", "--enable-dtlscid", "--enable-dtls13", + "--enable-secure-renegotiation", "--enable-psk", "--enable-aesccm", + "--enable-nullcipher", "CPPFLAGS=-DWOLFSSL_STATIC_RSA"]}, + {"name": "user-settings-all", "minutes": 1.5, + "comment": "The user_settings.h header-driven build path under Apple clang: macOS-specific guard ordering (e.g. WOLFSSL_SYS_CA_CERTS pulling in Security.framework) is distinct from the autotools --enable-all path above.", + "user_settings": "examples/configs/user_settings_all.h", + "cflags": "", + "configure": ["--enable-usersettings"]}, + {"name": "apple-native-cert-validation", "minutes": 1.5, + "comment": "Former macos-apple-native-cert-validation.yml workflow: WOLFSSL_APPLE_NATIVE_CERT_VALIDATION delegates chain verification to Security.framework instead of wolfSSL's verifier, and the TEST define enables its client tests in make check. CFLAGS go to configure (with \"cflags\": \"\" so the make-time --cflags do not override them), exactly like the old standalone job.", + "cflags": "", + "configure": ["CFLAGS=-DWOLFSSL_APPLE_NATIVE_CERT_VALIDATION -DWOLFSSL_TEST_APPLE_NATIVE_CERT_VALIDATION -DRSA_MIN_SIZE=2048 -DNO_WOLFSSL_CIPHER_SUITE_TEST"]}, + {"name": "cryptocb-keygen-utils-setkey", "minutes": 1.0, + "comment": "Crypto-callback dispatcher under Apple clang. Not covered by --enable-all; verifies the cryptocb find/setkey/keygen path compiles and runs on the macOS toolchain.", + "configure": ["--enable-cryptocb", "--enable-keygen", "--enable-cryptocbutils=setkey"]}, + {"name": "default", "minutes": 0.5, + "comment": "Default build: --enable-sys-ca-certs is auto-on on macOS, so this exercises Apple keychain / system trust loading in src/ssl_load.c that has no Linux equivalent.", + "configure": []}, + {"name": "no-sys-ca-certs", "minutes": 0.5, + "comment": "Validates the configure-time auto-enable override and that the build compiles out the Security.framework code path cleanly - macOS is the only OS where sys-ca-certs is auto-on by default.", + "configure": ["--disable-sys-ca-certs"]} + ] + EOF + .github/scripts/parallel-make-check.py \ + --threads 1 --jobs 3 --cc= \ + --cflags='-pedantic -Wdeclaration-after-statement -Wnull-dereference -Wno-overlength-strings -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE' \ + --private-dir=certs "$RUNNER_TEMP/os-check-macos-configs.json" + + - name: ccache stats + if: always() + run: ccache -s || true + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v4 with: - python-version: '3.x' - - run: pip install tlslite-ng - - - name: Build and test wolfSSL - uses: wolfSSL/actions-build-autotools-project@v1 - with: - configure: --enable-usersettings - check: true - user-settings: ${{ matrix.user-settings }} - - make_user_settings_testwolfcrypt: - # testwolfcrypt runs pure crypto tests with no platform-specific - # features, so Linux-only is sufficient for these user_settings. - strategy: - fail-fast: false - matrix: - user-settings: [ - # Add new user_settings.h here (alphabetical order) - 'examples/configs/user_settings_ca.h', - 'examples/configs/user_settings_dtls13.h', - 'examples/configs/user_settings_EBSnet.h', - 'examples/configs/user_settings_eccnonblock.h', - 'examples/configs/user_settings_curve25519nonblock.h', - 'examples/configs/user_settings_min_ecc.h', - 'examples/configs/user_settings_openssl_compat.h', - 'examples/configs/user_settings_pkcs7.h', - 'examples/configs/user_settings_rsa_only.h', - 'examples/configs/user_settings_template.h', - 'examples/configs/user_settings_tls12.h', - 'examples/configs/user_settings_tls13.h', - 'examples/configs/user_settings_wolfboot_keytools.h', - 'examples/configs/user_settings_wolfssh.h', - 'examples/configs/user_settings_wolftpm.h', - # Not included (require special setup): - # - user_settings_pq.h: Requires --enable-experimental - # - user_settings_baremetal.h: Requires static memory, custom platform - ] - name: make user_setting.h (testwolfcrypt only) - if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} - runs-on: ubuntu-24.04 - # This should be a safe limit for the tests to run. - timeout-minutes: 14 - steps: - - name: Build and test wolfSSL - uses: wolfSSL/actions-build-autotools-project@v1 - with: - configure: --enable-usersettings --disable-examples - check: false - user-settings: ${{ matrix.user-settings }} - - - name: Run wolfcrypt/test/testwolfcrypt - run: ./wolfcrypt/test/testwolfcrypt - - # Has to be dedicated function due to the sed call. - # Platform-agnostic; --enable-all macOS coverage in make_check_macos and - # the macOS user_settings_all.h run in make_user_settings already cover - # the equivalent code paths on Darwin. - make_user_all: - name: make user_setting.h (with sed) - if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} - runs-on: ubuntu-24.04 - # This should be a safe limit for the tests to run. - timeout-minutes: 14 - steps: - - uses: actions/checkout@v4 - # tlslite-ng is consumed by scripts/multi-msg-record.test (run from - # `make check`); without it that test is SKIPped. - - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - run: pip install tlslite-ng - - run: ./autogen.sh - - name: user_settings_all.h with compatibility layer - run: | - cp ./examples/configs/user_settings_all.h user_settings.h - sed -i -e "s/if 0/if 1/" user_settings.h - ./configure --enable-usersettings - make -j - make check + name: os-check-macos-logs + path: | + build-*/make-check.log + build-*/test-suite.log + build-*/config.log + if-no-files-found: ignore windows_build: name: Windows Build Test diff --git a/.github/workflows/pq-all.yml b/.github/workflows/pq-all.yml index 1d6e90b3e0..67a37cfbfb 100644 --- a/.github/workflows/pq-all.yml +++ b/.github/workflows/pq-all.yml @@ -14,57 +14,227 @@ concurrency: # END OF COMMON SECTION jobs: + # All former runner-per-config matrix entries build on one runner via + # .github/scripts/parallel-make-check.py (see os-check.yml for the full + # pattern): each config in its own out-of-tree ("VPATH") build directory + # off one checkout/autogen, checks on a pool of one-per-CPU worker + # threads, longest first. bubblewrap gives every test script its own + # network namespace so concurrent checks cannot collide on TCP/UDP ports + # (do not set AM_BWRAPPED here - that would disable it). make_check: - strategy: - matrix: - config: [ - # Add new configs here - '--disable-shared --enable-dilithium --enable-mlkem CFLAGS="-fsanitize=undefined -fno-sanitize-recover=undefined -fno-omit-frame-pointer" LDFLAGS="-fsanitize=undefined" CPPFLAGS="-DWOLFSSL_MLDSA_ALIGNMENT=4"', - '--enable-intelasm --enable-sp-asm --enable-mlkem=yes,kyber,ml-kem CPPFLAGS="-DWOLFSSL_ML_KEM_USE_OLD_IDS"', - '--enable-intelasm --enable-sp-asm --enable-all --enable-testcert --enable-acert --enable-dtls13 --enable-dtls-mtu --enable-dtls-frag-ch --enable-dtlscid --enable-quic --with-sys-crypto-policy --enable-experimental --enable-mlkem=yes,kyber,ml-kem --enable-tls-mlkem-standalone --enable-extra-pqc-hybrids --enable-lms --enable-xmss --enable-slhdsa --enable-dilithium --enable-dual-alg-certs --disable-qt CPPFLAGS="-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFSSL_BLIND_PRIVATE_KEY -DWOLFSSL_TLSX_PQC_MLKEM_STORE_OBJ"', - '--enable-intelasm --enable-sp-math --enable-sp-asm --enable-all --enable-testcert --enable-acert --enable-dtls13 --enable-dtls-mtu --enable-dtls-frag-ch --enable-dtlscid --disable-quic --with-sys-crypto-policy --enable-experimental --enable-mlkem=yes,kyber,ml-kem --enable-tls-mlkem-standalone --enable-extra-pqc-hybrids --enable-lms --enable-xmss --enable-slhdsa --enable-dilithium --enable-dual-alg-certs --disable-qt CPPFLAGS="-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFSSL_BLIND_PRIVATE_KEY -DWOLFSSL_TLSX_PQC_MLKEM_STORE_OBJ"', - '--enable-smallstack --enable-smallstackcache --enable-intelasm --enable-sp-asm --enable-all --enable-testcert --enable-acert --enable-dtls13 --enable-dtls-mtu --enable-dtls-frag-ch --enable-dtlscid --enable-quic --with-sys-crypto-policy --enable-experimental --enable-mlkem=yes,kyber,ml-kem --enable-lms --enable-xmss --enable-slhdsa --enable-dilithium --enable-dual-alg-certs --disable-qt CPPFLAGS="-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"', - '--enable-intelasm --enable-sp-asm --enable-all --enable-testcert --enable-acert --enable-dtls13 --enable-dtls-mtu --enable-dtls-frag-ch --enable-dtlscid --enable-quic --with-sys-crypto-policy --enable-experimental --enable-mlkem=yes,kyber,ml-kem --enable-lms --enable-xmss --enable-slhdsa --enable-dilithium --enable-dual-alg-certs --disable-qt CPPFLAGS="-Wdeclaration-after-statement -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE" CC=c++', - '--disable-intelasm --enable-all --enable-testcert --enable-acert --enable-dtls13 --enable-dtls-mtu --enable-dtls-frag-ch --enable-dtlscid --enable-quic --with-sys-crypto-policy --enable-experimental --enable-mlkem=yes,kyber,ml-kem,small --enable-lms=yes,small --enable-xmss=yes,small --enable-slhdsa=yes,small --enable-dilithium=yes,small --enable-dual-alg-certs --disable-qt CPPFLAGS="-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFSSL_MLKEM_MAKEKEY_SMALL_MEM -DWOLFSSL_MLKEM_ENCAPSULATE_SMALL_MEM -DWOLFSSL_MLKEM_NO_LARGE_CODE -DWOLFSSL_MLDSA_SIGN_SMALL_MEM -DWOLFSSL_MLDSA_VERIFY_SMALL_MEM -DWOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM -DWOLFSSL_MLDSA_NO_LARGE_CODE"', - '--disable-intelasm --enable-smallstack --enable-smallstackcache --enable-all --enable-testcert --enable-acert --enable-dtls13 --enable-dtls-mtu --enable-dtls-frag-ch --enable-dtlscid --enable-quic --with-sys-crypto-policy --enable-experimental --enable-mlkem=yes,kyber,ml-kem,small --enable-lms=yes,small --enable-xmss=yes,small --enable-slhdsa=yes,small --enable-dilithium=yes,small --enable-dual-alg-certs --disable-qt CPPFLAGS="-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFSSL_MLKEM_MAKEKEY_SMALL_MEM -DWOLFSSL_MLKEM_ENCAPSULATE_SMALL_MEM -DWOLFSSL_MLKEM_NO_LARGE_CODE -DWOLFSSL_MLDSA_SIGN_SMALL_MEM -DWOLFSSL_MLDSA_VERIFY_SMALL_MEM -DWOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM -DWOLFSSL_MLDSA_NO_LARGE_CODE"', - '--disable-intelasm --enable-all --disable-mlkem --enable-lms=yes,small,verify-only --enable-xmss=yes,small,verify-only --enable-slhdsa=yes,small,verify-only --enable-dilithium=yes,small,verify-only --disable-qt CPPFLAGS="-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFSSL_MLDSA_VERIFY_SMALL_MEM -DWOLFSSL_MLDSA_NO_LARGE_CODE"', - '--enable-intelasm --enable-sp-asm --enable-all --enable-testcert --enable-dtls13 --enable-dtls-mtu --enable-dtls-frag-ch --enable-dtlscid --enable-mlkem=make,enc,dec,512 --enable-tls-mlkem-standalone --disable-qt CPPFLAGS="-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"', - '--enable-intelasm --enable-sp-asm --enable-all --enable-testcert --enable-dtls13 --enable-dtls-mtu --enable-dtls-frag-ch --enable-dtlscid --enable-mlkem=make,enc,dec,768 --disable-qt CPPFLAGS="-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"', - '--enable-intelasm --enable-sp-asm --enable-all --enable-testcert --enable-dtls13 --enable-dtls-mtu --enable-dtls-frag-ch --enable-dtlscid --enable-mlkem=make,enc,dec,768 --enable-tls-mlkem-standalone --disable-qt CPPFLAGS="-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"', - '--enable-intelasm --enable-sp-asm --enable-all --enable-testcert --enable-dtls13 --enable-dtls-mtu --enable-dtls-frag-ch --enable-dtlscid --enable-mlkem=make,enc,dec,768 --enable-tls-mlkem-standalone --disable-pqc-hybrids --disable-qt CPPFLAGS="-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"', - '--enable-intelasm --enable-sp-asm --enable-all --enable-testcert --enable-dtls13 --enable-dtls-mtu --enable-dtls-frag-ch --enable-dtlscid --enable-mlkem=make,enc,dec,1024 --disable-qt CPPFLAGS="-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"', - '--enable-intelasm --enable-sp-asm --enable-all --enable-testcert --enable-dtls13 --enable-dtls-mtu --enable-dtls-frag-ch --enable-dtlscid --enable-mlkem=make,enc,dec,1024 --enable-tls-mlkem-standalone --disable-qt CPPFLAGS="-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"', - '--enable-intelasm --enable-sp-asm --enable-all --enable-testcert --enable-dtls13 --enable-dtls-mtu --enable-dtls-frag-ch --enable-dtlscid --enable-mlkem=make,enc,dec,1024 --enable-tls-mlkem-standalone --disable-pqc-hybrids --disable-qt CPPFLAGS="-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"', - '--enable-intelasm --enable-sp-asm --enable-all --enable-testcert --enable-acert --enable-dtls13 --enable-dtls-mtu --enable-dtls-frag-ch --enable-dtlscid --enable-quic --with-sys-crypto-policy --enable-experimental --enable-mlkem=yes,kyber,ml-kem --enable-lms --enable-xmss --enable-slhdsa --enable-dilithium=yes,no-ctx --enable-dual-alg-certs --disable-qt CPPFLAGS="-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"', - '--enable-intelasm --enable-sp-asm --enable-mlkem=yes,kyber,ml-kem,cache-a CPPFLAGS="-DWOLFSSL_MLKEM_DYNAMIC_KEYS"', - '--enable-intelasm --enable-sp-asm --enable-dilithium=yes CPPFLAGS="-DWOLFSSL_MLDSA_DYNAMIC_KEYS"', - '--disable-intelasm --enable-dilithium=yes,small CPPFLAGS="-DWOLFSSL_MLDSA_DYNAMIC_KEYS"', - '--disable-intelasm --enable-dilithium=44,65,87,verify-only CPPFLAGS="-DWOLFSSL_MLDSA_DYNAMIC_KEYS"', - ] name: make check + strategy: + fail-fast: false + matrix: + shard: [1, 2] if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} runs-on: ubuntu-24.04 - # This should be a safe limit for the tests to run. - timeout-minutes: 10 + # Generous for a cold ccache; warm reruns finish in a fraction. + timeout-minutes: 30 steps: - uses: actions/checkout@v4 name: Checkout wolfSSL - - name: Test wolfSSL - run: | - ./autogen.sh - ./configure ${{ matrix.config }} - make -j 4 - make check + - name: Install dependencies + uses: ./.github/actions/install-apt-deps + with: + packages: autoconf automake libtool build-essential bubblewrap - - name: Print errors - if: ${{ failure() }} + # ccache via the cross-platform composite; the script passes the + # compiler to configure as CC="ccache gcc" (or a per-config "cc"). + - name: Set up ccache + uses: ./.github/actions/ccache-setup + with: + workflow-id: pq-all + config-hash: shard-${{ matrix.shard }} + max-size: 350M + + # Ubuntu 24.04 can restrict unprivileged user namespaces via AppArmor, + # which would stop the test scripts from re-execing under + # bwrap --unshare-net (their port-isolation mechanism). + - name: Allow unprivileged user namespaces (for bwrap) + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true + + # The JSON list below is the former runner-per-config matrix; add new + # configs as new entries (a "comment" key is allowed for notes). + # "minutes" is the expected duration driving longest-first scheduling: + # take it from the Minutes column of a previous run's step summary, or + # omit it for a new config (defaults to 1) and refresh later. The list + # is kept sorted by minutes for readability, but the schedule sorts by + # the values, not list order. + - name: Build and make check this shard's configs (parallel, out-of-tree) run: | - for file in scripts/*.log - do - if [ -f "$file" ]; then - echo "${file}:" - cat "$file" - echo "========================================================================" - fi - done + cat > "$RUNNER_TEMP/pq-all-configs.json" <<'EOF' + [ + {"name": "all-pq-small-noasm", "minutes": 4, + "configure": ["--disable-intelasm", "--enable-all", + "--enable-testcert", "--enable-acert", "--enable-dtls13", + "--enable-dtls-mtu", "--enable-dtls-frag-ch", "--enable-dtlscid", + "--enable-quic", "--with-sys-crypto-policy", + "--enable-experimental", "--enable-mlkem=yes,kyber,ml-kem,small", + "--enable-lms=yes,small", "--enable-xmss=yes,small", + "--enable-slhdsa=yes,small", "--enable-dilithium=yes,small", + "--enable-dual-alg-certs", "--disable-qt", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFSSL_MLKEM_MAKEKEY_SMALL_MEM -DWOLFSSL_MLKEM_ENCAPSULATE_SMALL_MEM -DWOLFSSL_MLKEM_NO_LARGE_CODE -DWOLFSSL_MLDSA_SIGN_SMALL_MEM -DWOLFSSL_MLDSA_VERIFY_SMALL_MEM -DWOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM -DWOLFSSL_MLDSA_NO_LARGE_CODE"]}, + {"name": "all-pq-small-smallstack-noasm", "minutes": 4, + "configure": ["--disable-intelasm", "--enable-smallstack", + "--enable-smallstackcache", "--enable-all", "--enable-testcert", + "--enable-acert", "--enable-dtls13", "--enable-dtls-mtu", + "--enable-dtls-frag-ch", "--enable-dtlscid", "--enable-quic", + "--with-sys-crypto-policy", "--enable-experimental", + "--enable-mlkem=yes,kyber,ml-kem,small", "--enable-lms=yes,small", + "--enable-xmss=yes,small", "--enable-slhdsa=yes,small", + "--enable-dilithium=yes,small", "--enable-dual-alg-certs", + "--disable-qt", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFSSL_MLKEM_MAKEKEY_SMALL_MEM -DWOLFSSL_MLKEM_ENCAPSULATE_SMALL_MEM -DWOLFSSL_MLKEM_NO_LARGE_CODE -DWOLFSSL_MLDSA_SIGN_SMALL_MEM -DWOLFSSL_MLDSA_VERIFY_SMALL_MEM -DWOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM -DWOLFSSL_MLDSA_NO_LARGE_CODE"]}, + {"name": "all-pq-quic", "minutes": 3.5, + "configure": ["--enable-intelasm", "--enable-sp-asm", + "--enable-all", "--enable-testcert", "--enable-acert", + "--enable-dtls13", "--enable-dtls-mtu", "--enable-dtls-frag-ch", + "--enable-dtlscid", "--enable-quic", "--with-sys-crypto-policy", + "--enable-experimental", "--enable-mlkem=yes,kyber,ml-kem", + "--enable-tls-mlkem-standalone", "--enable-extra-pqc-hybrids", + "--enable-lms", "--enable-xmss", "--enable-slhdsa", + "--enable-dilithium", "--enable-dual-alg-certs", "--disable-qt", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFSSL_BLIND_PRIVATE_KEY -DWOLFSSL_TLSX_PQC_MLKEM_STORE_OBJ"]}, + {"name": "all-pq-sp-math", "minutes": 3.5, + "configure": ["--enable-intelasm", "--enable-sp-math", + "--enable-sp-asm", "--enable-all", "--enable-testcert", + "--enable-acert", "--enable-dtls13", "--enable-dtls-mtu", + "--enable-dtls-frag-ch", "--enable-dtlscid", "--disable-quic", + "--with-sys-crypto-policy", "--enable-experimental", + "--enable-mlkem=yes,kyber,ml-kem", "--enable-tls-mlkem-standalone", + "--enable-extra-pqc-hybrids", "--enable-lms", "--enable-xmss", + "--enable-slhdsa", "--enable-dilithium", "--enable-dual-alg-certs", + "--disable-qt", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFSSL_BLIND_PRIVATE_KEY -DWOLFSSL_TLSX_PQC_MLKEM_STORE_OBJ"]}, + {"name": "all-pq-smallstack", "minutes": 3.5, + "configure": ["--enable-smallstack", "--enable-smallstackcache", + "--enable-intelasm", "--enable-sp-asm", "--enable-all", + "--enable-testcert", "--enable-acert", "--enable-dtls13", + "--enable-dtls-mtu", "--enable-dtls-frag-ch", "--enable-dtlscid", + "--enable-quic", "--with-sys-crypto-policy", + "--enable-experimental", "--enable-mlkem=yes,kyber,ml-kem", + "--enable-lms", "--enable-xmss", "--enable-slhdsa", + "--enable-dilithium", "--enable-dual-alg-certs", "--disable-qt", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"]}, + {"name": "all-pq-cxx", "minutes": 3.5, + "cc": "ccache c++", + "configure": ["--enable-intelasm", "--enable-sp-asm", + "--enable-all", "--enable-testcert", "--enable-acert", + "--enable-dtls13", "--enable-dtls-mtu", "--enable-dtls-frag-ch", + "--enable-dtlscid", "--enable-quic", "--with-sys-crypto-policy", + "--enable-experimental", "--enable-mlkem=yes,kyber,ml-kem", + "--enable-lms", "--enable-xmss", "--enable-slhdsa", + "--enable-dilithium", "--enable-dual-alg-certs", "--disable-qt", + "CPPFLAGS=-Wdeclaration-after-statement -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"]}, + {"name": "all-pq-verify-only-noasm", "minutes": 3, + "configure": ["--disable-intelasm", "--enable-all", + "--disable-mlkem", "--enable-lms=yes,small,verify-only", + "--enable-xmss=yes,small,verify-only", + "--enable-slhdsa=yes,small,verify-only", + "--enable-dilithium=yes,small,verify-only", "--disable-qt", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE -DWOLFSSL_MLDSA_VERIFY_SMALL_MEM -DWOLFSSL_MLDSA_NO_LARGE_CODE"]}, + {"name": "all-pq-mldsa-no-ctx", "minutes": 3, + "configure": ["--enable-intelasm", "--enable-sp-asm", + "--enable-all", "--enable-testcert", "--enable-acert", + "--enable-dtls13", "--enable-dtls-mtu", "--enable-dtls-frag-ch", + "--enable-dtlscid", "--enable-quic", "--with-sys-crypto-policy", + "--enable-experimental", "--enable-mlkem=yes,kyber,ml-kem", + "--enable-lms", "--enable-xmss", "--enable-slhdsa", + "--enable-dilithium=yes,no-ctx", "--enable-dual-alg-certs", + "--disable-qt", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"]}, + {"name": "ubsan-mldsa-mlkem", "minutes": 2.5, + "configure": ["--disable-shared", "--enable-dilithium", + "--enable-mlkem", + "CFLAGS=-fsanitize=undefined -fno-sanitize-recover=undefined -fno-omit-frame-pointer", + "LDFLAGS=-fsanitize=undefined", + "CPPFLAGS=-DWOLFSSL_MLDSA_ALIGNMENT=4"]}, + {"name": "all-mlkem-512-standalone", "minutes": 2.5, + "configure": ["--enable-intelasm", "--enable-sp-asm", + "--enable-all", "--enable-testcert", "--enable-dtls13", + "--enable-dtls-mtu", "--enable-dtls-frag-ch", "--enable-dtlscid", + "--enable-mlkem=make,enc,dec,512", "--enable-tls-mlkem-standalone", + "--disable-qt", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"]}, + {"name": "all-mlkem-768", "minutes": 2.5, + "configure": ["--enable-intelasm", "--enable-sp-asm", + "--enable-all", "--enable-testcert", "--enable-dtls13", + "--enable-dtls-mtu", "--enable-dtls-frag-ch", "--enable-dtlscid", + "--enable-mlkem=make,enc,dec,768", "--disable-qt", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"]}, + {"name": "all-mlkem-768-standalone", "minutes": 2.5, + "configure": ["--enable-intelasm", "--enable-sp-asm", + "--enable-all", "--enable-testcert", "--enable-dtls13", + "--enable-dtls-mtu", "--enable-dtls-frag-ch", "--enable-dtlscid", + "--enable-mlkem=make,enc,dec,768", "--enable-tls-mlkem-standalone", + "--disable-qt", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"]}, + {"name": "all-mlkem-768-standalone-no-hybrids", "minutes": 2.5, + "configure": ["--enable-intelasm", "--enable-sp-asm", + "--enable-all", "--enable-testcert", "--enable-dtls13", + "--enable-dtls-mtu", "--enable-dtls-frag-ch", "--enable-dtlscid", + "--enable-mlkem=make,enc,dec,768", "--enable-tls-mlkem-standalone", + "--disable-pqc-hybrids", "--disable-qt", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"]}, + {"name": "all-mlkem-1024", "minutes": 2.5, + "configure": ["--enable-intelasm", "--enable-sp-asm", + "--enable-all", "--enable-testcert", "--enable-dtls13", + "--enable-dtls-mtu", "--enable-dtls-frag-ch", "--enable-dtlscid", + "--enable-mlkem=make,enc,dec,1024", "--disable-qt", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"]}, + {"name": "all-mlkem-1024-standalone", "minutes": 2.5, + "configure": ["--enable-intelasm", "--enable-sp-asm", + "--enable-all", "--enable-testcert", "--enable-dtls13", + "--enable-dtls-mtu", "--enable-dtls-frag-ch", "--enable-dtlscid", + "--enable-mlkem=make,enc,dec,1024", + "--enable-tls-mlkem-standalone", "--disable-qt", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"]}, + {"name": "all-mlkem-1024-standalone-no-hybrids", "minutes": 2.5, + "configure": ["--enable-intelasm", "--enable-sp-asm", + "--enable-all", "--enable-testcert", "--enable-dtls13", + "--enable-dtls-mtu", "--enable-dtls-frag-ch", "--enable-dtlscid", + "--enable-mlkem=make,enc,dec,1024", + "--enable-tls-mlkem-standalone", "--disable-pqc-hybrids", + "--disable-qt", + "CPPFLAGS=-pedantic -Wdeclaration-after-statement -Wnull-dereference -DWOLFCRYPT_TEST_LINT -DNO_WOLFSSL_CIPHER_SUITE_TEST -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE"]}, + {"name": "mlkem-old-ids", "minutes": 1.2, + "configure": ["--enable-intelasm", "--enable-sp-asm", + "--enable-mlkem=yes,kyber,ml-kem", + "CPPFLAGS=-DWOLFSSL_ML_KEM_USE_OLD_IDS"]}, + {"name": "mlkem-dynamic-keys", "minutes": 1.2, + "configure": ["--enable-intelasm", "--enable-sp-asm", + "--enable-mlkem=yes,kyber,ml-kem,cache-a", + "CPPFLAGS=-DWOLFSSL_MLKEM_DYNAMIC_KEYS"]}, + {"name": "mldsa-dynamic-keys", "minutes": 1.2, + "configure": ["--enable-intelasm", "--enable-sp-asm", + "--enable-dilithium=yes", "CPPFLAGS=-DWOLFSSL_MLDSA_DYNAMIC_KEYS"]}, + {"name": "mldsa-small-dynamic-keys", "minutes": 1.2, + "configure": ["--disable-intelasm", "--enable-dilithium=yes,small", + "CPPFLAGS=-DWOLFSSL_MLDSA_DYNAMIC_KEYS"]}, + {"name": "mldsa-verify-only-dynamic-keys", "minutes": 1.2, + "configure": ["--disable-intelasm", + "--enable-dilithium=44,65,87,verify-only", + "CPPFLAGS=-DWOLFSSL_MLDSA_DYNAMIC_KEYS"]} + ] + EOF + .github/scripts/parallel-make-check.py \ + --shard "${{ matrix.shard }}/${{ strategy.job-total }}" \ + --private-dir=certs \ + "$RUNNER_TEMP/pq-all-configs.json" + + - name: ccache stats + if: always() + run: ccache -s || true + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: pq-all-logs-${{ matrix.shard }} + path: | + build-*/make-check.log + build-*/test-suite.log + build-*/config.log + if-no-files-found: ignore diff --git a/.github/workflows/psk.yml b/.github/workflows/psk.yml index 05d6a80971..2bb512cb5b 100644 --- a/.github/workflows/psk.yml +++ b/.github/workflows/psk.yml @@ -14,41 +14,101 @@ concurrency: # END OF COMMON SECTION jobs: + # All former runner-per-config matrix entries build on one runner via + # .github/scripts/parallel-make-check.py (see os-check.yml for the full + # pattern): each config in its own out-of-tree ("VPATH") build directory + # off one checkout/autogen, checks on a pool of one-per-CPU worker + # threads, longest first. bubblewrap gives every test script its own + # network namespace so concurrent checks cannot collide on TCP/UDP ports + # (do not set AM_BWRAPPED here - that would disable it). make_check: - strategy: - matrix: - config: [ - # Add new configs here - '--enable-psk --enable-cert-with-extern-psk --disable-mlkem', - '--enable-psk --disable-mlkem C_EXTRA_FLAGS="-DWOLFSSL_STATIC_PSK -DWOLFSSL_OLDTLS_SHA2_CIPHERSUITES"', - '--enable-psk --disable-mlkem C_EXTRA_FLAGS=-DWOLFSSL_STATIC_PSK --disable-rsa --disable-ecc --disable-dh', - '--disable-oldtls --disable-tls13 --enable-psk -disable-rsa --disable-dh -disable-ecc --disable-asn C_EXTRA_FLAGS=-DWOLFSSL_STATIC_PSK --enable-lowresource --enable-singlethreaded --disable-asm --disable-errorstrings --disable-pkcs12 --disable-sha3 --disable-sha224 --disable-sha384 --disable-sha512 --disable-sha --disable-md5 -disable-aescbc --disable-chacha --disable-poly1305 --disable-coding --disable-sp-math-all --disable-mlkem', - '--disable-oldtls --disable-tlsv12 --enable-tls13 --enable-psk -disable-rsa --disable-dh -disable-ecc --disable-asn C_EXTRA_FLAGS=-DWOLFSSL_STATIC_PSK --enable-lowresource --enable-singlethreaded --disable-asm --disable-errorstrings --disable-pkcs12 --disable-sha3 --disable-sha224 --disable-sha384 --disable-sha512 --disable-sha --disable-md5 -disable-aescbc --disable-chacha --disable-poly1305 --disable-coding --disable-sp-math-all --disable-mlkem' - ] name: make check if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} runs-on: ubuntu-24.04 - # This should be a safe limit for the tests to run. - timeout-minutes: 6 + # Generous for a cold ccache; warm reruns finish in a fraction. + timeout-minutes: 10 steps: - uses: actions/checkout@v4 name: Checkout wolfSSL - - name: Test wolfSSL - run: | - ./autogen.sh - ./configure ${{ matrix.config }} - make -j 4 - make check + - name: Install dependencies + uses: ./.github/actions/install-apt-deps + with: + packages: autoconf automake libtool build-essential bubblewrap - - name: Print errors - if: ${{ failure() }} + # ccache via the cross-platform composite; the script passes the + # compiler to configure as CC="ccache gcc" (or a per-config "cc"). + - name: Set up ccache + uses: ./.github/actions/ccache-setup + with: + workflow-id: psk + max-size: 100M + + # Ubuntu 24.04 can restrict unprivileged user namespaces via AppArmor, + # which would stop the test scripts from re-execing under + # bwrap --unshare-net (their port-isolation mechanism). + - name: Allow unprivileged user namespaces (for bwrap) + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true + + # The JSON list below is the former runner-per-config matrix; add new + # configs as new entries (a "comment" key is allowed for notes). + # "minutes" is the expected duration driving longest-first scheduling: + # take it from the Minutes column of a previous run's step summary, or + # omit it for a new config (defaults to 1) and refresh later. The list + # is kept sorted by minutes for readability, but the schedule sorts by + # the values, not list order. + - name: Build and make check all configs (parallel, out-of-tree) run: | - for file in scripts/*.log - do - if [ -f "$file" ]; then - echo "${file}:" - cat "$file" - echo "========================================================================" - fi - done + cat > "$RUNNER_TEMP/psk-configs.json" <<'EOF' + [ + {"name": "psk-cert-extern", "minutes": 1, + "configure": ["--enable-psk", "--enable-cert-with-extern-psk", + "--disable-mlkem"]}, + {"name": "static-psk-oldtls-sha2", "minutes": 1, + "configure": ["--enable-psk", "--disable-mlkem", + "C_EXTRA_FLAGS=-DWOLFSSL_STATIC_PSK -DWOLFSSL_OLDTLS_SHA2_CIPHERSUITES"]}, + {"name": "static-psk-no-pk", "minutes": 1, + "configure": ["--enable-psk", "--disable-mlkem", + "C_EXTRA_FLAGS=-DWOLFSSL_STATIC_PSK", "--disable-rsa", + "--disable-ecc", "--disable-dh"]}, + {"name": "static-psk-lowresource-tls12", "minutes": 0.8, + "configure": ["--disable-oldtls", "--disable-tls13", "--enable-psk", + "-disable-rsa", "--disable-dh", "-disable-ecc", "--disable-asn", + "C_EXTRA_FLAGS=-DWOLFSSL_STATIC_PSK", "--enable-lowresource", + "--enable-singlethreaded", "--disable-asm", + "--disable-errorstrings", "--disable-pkcs12", "--disable-sha3", + "--disable-sha224", "--disable-sha384", "--disable-sha512", + "--disable-sha", "--disable-md5", "-disable-aescbc", + "--disable-chacha", "--disable-poly1305", "--disable-coding", + "--disable-sp-math-all", "--disable-mlkem"]}, + {"name": "static-psk-lowresource-tls13", "minutes": 0.8, + "configure": ["--disable-oldtls", "--disable-tlsv12", + "--enable-tls13", "--enable-psk", "-disable-rsa", "--disable-dh", + "-disable-ecc", "--disable-asn", + "C_EXTRA_FLAGS=-DWOLFSSL_STATIC_PSK", "--enable-lowresource", + "--enable-singlethreaded", "--disable-asm", + "--disable-errorstrings", "--disable-pkcs12", "--disable-sha3", + "--disable-sha224", "--disable-sha384", "--disable-sha512", + "--disable-sha", "--disable-md5", "-disable-aescbc", + "--disable-chacha", "--disable-poly1305", "--disable-coding", + "--disable-sp-math-all", "--disable-mlkem"]} + ] + EOF + .github/scripts/parallel-make-check.py \ + --private-dir=certs \ + "$RUNNER_TEMP/psk-configs.json" + + - name: ccache stats + if: always() + run: ccache -s || true + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: psk-logs + path: | + build-*/make-check.log + build-*/test-suite.log + build-*/config.log + if-no-files-found: ignore diff --git a/.github/workflows/smallStackSize.yml b/.github/workflows/smallStackSize.yml index 286bb4527c..8717862ea4 100644 --- a/.github/workflows/smallStackSize.yml +++ b/.github/workflows/smallStackSize.yml @@ -14,41 +14,123 @@ concurrency: # END OF COMMON SECTION jobs: + # All former runner-per-config matrix entries build on one runner via + # .github/scripts/parallel-make-check.py (see os-check.yml for the full + # pattern): each config builds in its own out-of-tree ("VPATH") build + # directory off one checkout/autogen, on a pool of one-per-CPU worker + # threads, longest first. build_library: - strategy: - matrix: - config: [ - # defaults, noasm - '--disable-asm', - - # defaults + native PQ, no asm - '--disable-asm --enable-mlkem --enable-lms --enable-xmss --enable-mldsa', - - # all-crypto + native PQ, no asm - '--disable-asm --enable-all-crypto --enable-mlkem --enable-lms --enable-xmss --enable-mldsa', - - # defaults, intelasm + sp-asm - '--enable-intelasm --enable-sp-asm', - - # defaults + native PQ, intelasm + sp-asm - '--enable-intelasm --enable-sp-asm --enable-mlkem --enable-lms --enable-xmss --enable-mldsa', - - # all-crypto + native PQ, intelasm + sp-asm - '--enable-intelasm --enable-sp-asm --enable-all-crypto --enable-mlkem --enable-lms --enable-xmss --enable-mldsa' - ] name: build library if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} runs-on: ubuntu-24.04 - # This should be a safe limit for the tests to run. - timeout-minutes: 6 + # Generous for a cold ccache; warm reruns finish in a fraction. + timeout-minutes: 15 steps: - uses: actions/checkout@v4 name: Checkout wolfSSL - - name: Build wolfCrypt with smallstack and stack depth warnings, and run testwolfcrypt + - name: Install dependencies + uses: ./.github/actions/install-apt-deps + with: + packages: autoconf automake libtool build-essential + + # ccache via the cross-platform composite; the script passes the + # compiler to configure as CC="ccache gcc" (or a per-config "cc"). + - name: Set up ccache + uses: ./.github/actions/ccache-setup + with: + workflow-id: smallstacksize + max-size: 150M + + # The JSON list below is the former runner-per-config matrix (the + # shared base configure arguments are folded into every entry). Each + # build must come out clean under -Wframe-larger-than/-Wstack-usage, + # then runs testwolfcrypt under the relative-stack checker. + - name: Build all configs (parallel, out-of-tree) run: | - ./autogen.sh || $(exit 2) - echo "running ./configure ... ${{ matrix.config }}" - ./configure --enable-cryptonly --disable-cryptocb --disable-testcert --enable-smallstack --enable-smallstackcache --enable-crypttests --disable-benchmark --disable-examples --with-max-rsa-bits=16384 --enable-stacksize=verbose CFLAGS="-Wframe-larger-than=2048 -Wstack-usage=4096 -DWOLFSSL_TEST_MAX_RELATIVE_STACK_BYTES=8192 -DTEST_ALWAYS_RUN_TO_END" ${{ matrix.config }} || $(exit 3) - make -j 4 || $(exit 4) - ./wolfcrypt/test/testwolfcrypt + cat > "$RUNNER_TEMP/smallstacksize-configs.json" <<'EOF' + [ + {"name": "noasm", "minutes": 1, + "configure": ["--enable-cryptonly", "--disable-cryptocb", + "--disable-testcert", "--enable-smallstack", + "--enable-smallstackcache", "--enable-crypttests", + "--disable-benchmark", "--disable-examples", + "--with-max-rsa-bits=16384", "--enable-stacksize=verbose", + "CFLAGS=-Wframe-larger-than=2048 -Wstack-usage=4096 -DWOLFSSL_TEST_MAX_RELATIVE_STACK_BYTES=8192 -DTEST_ALWAYS_RUN_TO_END", + "--disable-asm"], + "check": false, + "run": [["./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "noasm-pq", "minutes": 1, + "configure": ["--enable-cryptonly", "--disable-cryptocb", + "--disable-testcert", "--enable-smallstack", + "--enable-smallstackcache", "--enable-crypttests", + "--disable-benchmark", "--disable-examples", + "--with-max-rsa-bits=16384", "--enable-stacksize=verbose", + "CFLAGS=-Wframe-larger-than=2048 -Wstack-usage=4096 -DWOLFSSL_TEST_MAX_RELATIVE_STACK_BYTES=8192 -DTEST_ALWAYS_RUN_TO_END", + "--disable-asm", "--enable-mlkem", "--enable-lms", "--enable-xmss", + "--enable-mldsa"], + "check": false, + "run": [["./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "noasm-allcrypto-pq", "minutes": 1, + "configure": ["--enable-cryptonly", "--disable-cryptocb", + "--disable-testcert", "--enable-smallstack", + "--enable-smallstackcache", "--enable-crypttests", + "--disable-benchmark", "--disable-examples", + "--with-max-rsa-bits=16384", "--enable-stacksize=verbose", + "CFLAGS=-Wframe-larger-than=2048 -Wstack-usage=4096 -DWOLFSSL_TEST_MAX_RELATIVE_STACK_BYTES=8192 -DTEST_ALWAYS_RUN_TO_END", + "--disable-asm", "--enable-all-crypto", "--enable-mlkem", + "--enable-lms", "--enable-xmss", "--enable-mldsa"], + "check": false, + "run": [["./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "intelasm", "minutes": 1, + "configure": ["--enable-cryptonly", "--disable-cryptocb", + "--disable-testcert", "--enable-smallstack", + "--enable-smallstackcache", "--enable-crypttests", + "--disable-benchmark", "--disable-examples", + "--with-max-rsa-bits=16384", "--enable-stacksize=verbose", + "CFLAGS=-Wframe-larger-than=2048 -Wstack-usage=4096 -DWOLFSSL_TEST_MAX_RELATIVE_STACK_BYTES=8192 -DTEST_ALWAYS_RUN_TO_END", + "--enable-intelasm", "--enable-sp-asm"], + "check": false, + "run": [["./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "intelasm-pq", "minutes": 1, + "configure": ["--enable-cryptonly", "--disable-cryptocb", + "--disable-testcert", "--enable-smallstack", + "--enable-smallstackcache", "--enable-crypttests", + "--disable-benchmark", "--disable-examples", + "--with-max-rsa-bits=16384", "--enable-stacksize=verbose", + "CFLAGS=-Wframe-larger-than=2048 -Wstack-usage=4096 -DWOLFSSL_TEST_MAX_RELATIVE_STACK_BYTES=8192 -DTEST_ALWAYS_RUN_TO_END", + "--enable-intelasm", "--enable-sp-asm", "--enable-mlkem", + "--enable-lms", "--enable-xmss", "--enable-mldsa"], + "check": false, + "run": [["./wolfcrypt/test/testwolfcrypt"]]}, + {"name": "intelasm-allcrypto-pq", "minutes": 1, + "configure": ["--enable-cryptonly", "--disable-cryptocb", + "--disable-testcert", "--enable-smallstack", + "--enable-smallstackcache", "--enable-crypttests", + "--disable-benchmark", "--disable-examples", + "--with-max-rsa-bits=16384", "--enable-stacksize=verbose", + "CFLAGS=-Wframe-larger-than=2048 -Wstack-usage=4096 -DWOLFSSL_TEST_MAX_RELATIVE_STACK_BYTES=8192 -DTEST_ALWAYS_RUN_TO_END", + "--enable-intelasm", "--enable-sp-asm", "--enable-all-crypto", + "--enable-mlkem", "--enable-lms", "--enable-xmss", + "--enable-mldsa"], + "check": false, + "run": [["./wolfcrypt/test/testwolfcrypt"]]} + ] + EOF + .github/scripts/parallel-make-check.py \ + "$RUNNER_TEMP/smallstacksize-configs.json" + + - name: ccache stats + if: always() + run: ccache -s || true + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: smallstacksize-logs + path: | + build-*/make-check.log + build-*/test-suite.log + build-*/config.log + if-no-files-found: ignore diff --git a/.github/workflows/smoke-test.yml b/.github/workflows/smoke-test.yml index 8c2f875c09..3ab7bf6ab5 100644 --- a/.github/workflows/smoke-test.yml +++ b/.github/workflows/smoke-test.yml @@ -5,13 +5,21 @@ name: Smoke Test # too: this is the gate that protects the rest of CI. Other PR workflows # wait for this via .github/actions/wait-for-smoke. # -# CFLAGS=-Werror is applied at make time only (not ./configure) so autoconf -# feature detection is not poisoned by benign warnings in conftest probes. +# The smoke config list lives in the "Build and make check" step below; +# the generic runner .github/scripts/parallel-make-check.py builds each +# config in its own out-of-tree ("VPATH") build directory off this single +# checkout and runs make check across them on a pool of one-per-CPU worker +# threads, reporting thread/CPU efficiency in the step summary. bubblewrap +# is installed so the script tests re-exec themselves under bwrap +# --unshare-net and concurrent checks cannot collide on TCP/UDP ports (do +# not set AM_BWRAPPED here - that would disable it). Builds go through +# ccache (cached across runs) to keep the single-runner job fast on warm +# caches. # # For pull_request events the workflow tests the POST-MERGE tree: # the PR head is checked out, the base branch is merged in, and: # * a merge conflict fails the job before any build runs. -# * if the PR tree is identical to base (no diff), the matrix is skipped. +# * if the PR tree is identical to base (no diff), the build is skipped. # * otherwise the build runs against the merged tree. # This catches stale PRs whose head builds clean but whose merge with # current master would break. @@ -38,38 +46,12 @@ permissions: jobs: smoke: + # Only run from the wolfssl org to avoid burning forks' CI minutes. + if: github.repository_owner == 'wolfssl' runs-on: ubuntu-24.04 - timeout-minutes: 25 - strategy: - fail-fast: false - matrix: - config: - - name: default - args: "" - - name: enable-all - args: "--enable-all" - - name: opensslextra - args: "--enable-opensslextra" - - name: enable-all-smallstack - args: "--enable-all --enable-smallstack" - - name: cryptonly - args: "--enable-cryptonly" - # Below entries target the top Jenkins PRB failure modes - # (-Werror unused-function / implicit-decl / link errors). - - name: leantls-extra - args: "--enable-leantls --enable-session-ticket --enable-sni --enable-opensslextra" - - name: dtls-suite - args: "--enable-psk --enable-dtls --enable-dtls13 --enable-dtls-mtu --enable-aesccm --enable-opensslextra" - - name: integration - args: "--enable-openssh --enable-lighty --enable-stunnel --enable-opensslextra" - # AddressSanitizer (UBSAN excluded - current master has known - # left-shift UB in auto-generated SP math). - - name: sanitize-asan - args: "--enable-all" - cflags: "-fsanitize=address -fno-omit-frame-pointer -g -O1" - ldflags: "-fsanitize=address" + timeout-minutes: 60 env: - MAKE_CFLAGS: "-Werror" + CCACHE_MAXSIZE: 2G steps: # For PRs we explicitly check out the PR head (not the auto-merge # ref) and do the merge ourselves below so we can fail fast on @@ -107,30 +89,77 @@ jobs: if: steps.merge_check.outputs.skip != 'true' uses: ./.github/actions/install-apt-deps with: - packages: autoconf automake libtool build-essential + packages: autoconf automake libtool build-essential bubblewrap ccache + + # Ubuntu 24.04 can restrict unprivileged user namespaces via AppArmor, + # which would stop the test scripts from re-execing under + # bwrap --unshare-net (their port-isolation mechanism). + - name: Allow unprivileged user namespaces (for bwrap) + if: steps.merge_check.outputs.skip != 'true' + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true + + - name: Restore ccache + if: steps.merge_check.outputs.skip != 'true' + uses: actions/cache@v4 + with: + path: ~/.cache/ccache + key: smoke-ccache-${{ github.base_ref || github.ref_name }}-${{ github.sha }} + restore-keys: | + smoke-ccache-${{ github.base_ref || github.ref_name }}- + smoke-ccache- - name: autogen if: steps.merge_check.outputs.skip != 'true' - run: ./autogen.sh - - - name: configure ${{ matrix.config.name }} - if: steps.merge_check.outputs.skip != 'true' - run: ./configure ${{ matrix.config.args }} - - - name: make - if: steps.merge_check.outputs.skip != 'true' - env: - ENTRY_CFLAGS: ${{ matrix.config.cflags }} - ENTRY_LDFLAGS: ${{ matrix.config.ldflags }} run: | - FLAGS="${ENTRY_CFLAGS:-$MAKE_CFLAGS}" - make -j"$(nproc)" CFLAGS="$FLAGS" LDFLAGS="$ENTRY_LDFLAGS" + ccache -z + ./autogen.sh - - name: make check + # Common-failure configs derived from the Jenkins PRB top-10 (last 30 + # days); leantls-extra, dtls-suite and integration target the top + # failure modes (-Werror unused-function / implicit-decl / link + # errors). Every config builds with -Werror unless it sets its own + # cflags: sanitize-asan replaces it with AddressSanitizer flags (UBSAN + # excluded - current master has known left-shift UB in auto-generated + # SP math). --private-dir=certs gives every build dir its own certs/ + # copy: crl-gen-openssl.test writes generated CRLs under certs/crl/, + # which would race through the shared VPATH certs symlink. + # + # List order is schedule order: the worker threads take configs from + # the top, so keep the slowest first or they straggle at the end on an + # otherwise idle runner. Order by the Minutes column of the step + # summary from a recent (warm-cache) run. + - name: Build and make check all configs (parallel, out-of-tree) if: steps.merge_check.outputs.skip != 'true' - env: - ENTRY_CFLAGS: ${{ matrix.config.cflags }} - ENTRY_LDFLAGS: ${{ matrix.config.ldflags }} run: | - FLAGS="${ENTRY_CFLAGS:-$MAKE_CFLAGS}" - make check CFLAGS="$FLAGS" LDFLAGS="$ENTRY_LDFLAGS" + cat > "$RUNNER_TEMP/smoke-configs.json" <<'EOF' + [ + {"name": "sanitize-asan", "configure": ["--enable-all"], + "cflags": "-fsanitize=address -fno-omit-frame-pointer -g -O1", + "ldflags": "-fsanitize=address"}, + {"name": "enable-all-smallstack", "configure": ["--enable-all", "--enable-smallstack"]}, + {"name": "enable-all", "configure": ["--enable-all"]}, + {"name": "integration", "configure": ["--enable-openssh", "--enable-lighty", "--enable-stunnel", "--enable-opensslextra"]}, + {"name": "dtls-suite", "configure": ["--enable-psk", "--enable-dtls", "--enable-dtls13", "--enable-dtls-mtu", "--enable-aesccm", "--enable-opensslextra"]}, + {"name": "opensslextra", "configure": ["--enable-opensslextra"]}, + {"name": "default"}, + {"name": "cryptonly", "configure": ["--enable-cryptonly"]}, + {"name": "leantls-extra", "configure": ["--enable-leantls", "--enable-session-ticket", "--enable-sni", "--enable-opensslextra"]} + ] + EOF + .github/scripts/parallel-make-check.py --cflags=-Werror \ + --private-dir=certs "$RUNNER_TEMP/smoke-configs.json" + + - name: ccache stats + if: always() && steps.merge_check.outputs.skip != 'true' + run: ccache -s || true + + - name: Upload logs on failure + if: failure() && steps.merge_check.outputs.skip != 'true' + uses: actions/upload-artifact@v4 + with: + name: smoke-logs + path: | + build-*/make-check.log + build-*/test-suite.log + build-*/config.log + if-no-files-found: ignore diff --git a/.github/workflows/trackmemory.yml b/.github/workflows/trackmemory.yml index f00ea8a4cb..ea04c96d82 100644 --- a/.github/workflows/trackmemory.yml +++ b/.github/workflows/trackmemory.yml @@ -14,48 +14,97 @@ concurrency: # END OF COMMON SECTION jobs: + # All former runner-per-config matrix entries build on one runner via + # .github/scripts/parallel-make-check.py (see os-check.yml for the full + # pattern): each config in its own out-of-tree ("VPATH") build directory + # off one checkout/autogen, checks on a pool of one-per-CPU worker + # threads, longest first. bubblewrap gives every test script its own + # network namespace so concurrent checks cannot collide on TCP/UDP ports + # (do not set AM_BWRAPPED here - that would disable it). make_check: - strategy: - matrix: - config: [ - # Add new configs here - '--enable-all --enable-debug-trace-errcodes CFLAGS="-DWC_RNG_SEED_CB -DWOLFSSL_TRACK_MEMORY -DWOLFSSL_DEBUG_MEMORY"', - '--enable-smallstack --enable-all CFLAGS="-DWC_RNG_SEED_CB -DWOLFSSL_TRACK_MEMORY -DWOLFSSL_DEBUG_MEMORY"', - '--enable-smallstackcache --enable-smallstack --enable-all --enable-debug-trace-errcodes CFLAGS="-DWC_RNG_SEED_CB -DWOLFSSL_TRACK_MEMORY -DWOLFSSL_DEBUG_MEMORY"', -# Note the below smallstackcache tests are crucial coverage for the Linux kernel -# module, when targeting a kernel with the randomness patch (linuxkm/patches/) -# applied. -# -# Note, don't combine wolfEntropy with the full TLS cipher suite test -- the implicit wc_InitRng()s in each suite have an enormous CPU footprint. - '--enable-wolfEntropy --enable-smallstackcache --enable-smallstack --enable-all CFLAGS="-DWC_RNG_SEED_CB -DWOLFSSL_TRACK_MEMORY -DWOLFSSL_DEBUG_MEMORY -DNO_WOLFSSL_CIPHER_SUITE_TEST"', - '--enable-intelrdseed --enable-smallstackcache --enable-smallstack --enable-all CFLAGS="-DWC_RNG_SEED_CB -DWOLFSSL_TRACK_MEMORY -DWOLFSSL_DEBUG_MEMORY -DNO_WOLFSSL_CIPHER_SUITE_TEST"', - '--enable-amdrand --enable-smallstackcache --enable-smallstack --enable-all CFLAGS="-DWC_RNG_SEED_CB -DWOLFSSL_TRACK_MEMORY -DWOLFSSL_DEBUG_MEMORY -DNO_WOLFSSL_CIPHER_SUITE_TEST"', - '--disable-asm --enable-wolfEntropy --enable-smallstackcache --enable-smallstack --enable-all CFLAGS="-DWC_RNG_SEED_CB -DWOLFSSL_TRACK_MEMORY -DWOLFSSL_DEBUG_MEMORY -DNO_WOLFSSL_CIPHER_SUITE_TEST"' - ] name: make check if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} runs-on: ubuntu-24.04 - # This should be a safe limit for the tests to run. - timeout-minutes: 6 + # Generous for a cold ccache; warm reruns finish in a fraction. + timeout-minutes: 20 steps: - uses: actions/checkout@v4 name: Checkout wolfSSL - - name: Test wolfSSL - run: | - ./autogen.sh - ./configure ${{ matrix.config }} - make -j 4 - make check + - name: Install dependencies + uses: ./.github/actions/install-apt-deps + with: + packages: autoconf automake libtool build-essential bubblewrap - - name: Print errors - if: ${{ failure() }} + # ccache via the cross-platform composite; the script passes the + # compiler to configure as CC="ccache gcc" (or a per-config "cc"). + - name: Set up ccache + uses: ./.github/actions/ccache-setup + with: + workflow-id: trackmemory + max-size: 250M + + # Ubuntu 24.04 can restrict unprivileged user namespaces via AppArmor, + # which would stop the test scripts from re-execing under + # bwrap --unshare-net (their port-isolation mechanism). + - name: Allow unprivileged user namespaces (for bwrap) + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true + + # The JSON list below is the former runner-per-config matrix; add new + # configs as new entries (a "comment" key is allowed for notes). + # "minutes" is the expected duration driving longest-first scheduling: + # take it from the Minutes column of a previous run's step summary, or + # omit it for a new config (defaults to 1) and refresh later. The list + # is kept sorted by minutes for readability, but the schedule sorts by + # the values, not list order. + - name: Build and make check all configs (parallel, out-of-tree) run: | - for file in scripts/*.log - do - if [ -f "$file" ]; then - echo "${file}:" - cat "$file" - echo "========================================================================" - fi - done + cat > "$RUNNER_TEMP/trackmemory-configs.json" <<'EOF' + [ + {"name": "all-noasm-wolfentropy", "minutes": 3, + "configure": ["--disable-asm", "--enable-wolfEntropy", + "--enable-smallstackcache", "--enable-smallstack", "--enable-all", + "CFLAGS=-DWC_RNG_SEED_CB -DWOLFSSL_TRACK_MEMORY -DWOLFSSL_DEBUG_MEMORY -DNO_WOLFSSL_CIPHER_SUITE_TEST"]}, + {"name": "all-trace-errcodes", "minutes": 2.5, + "configure": ["--enable-all", "--enable-debug-trace-errcodes", + "CFLAGS=-DWC_RNG_SEED_CB -DWOLFSSL_TRACK_MEMORY -DWOLFSSL_DEBUG_MEMORY"]}, + {"name": "all-smallstack", "minutes": 2.5, + "configure": ["--enable-smallstack", "--enable-all", + "CFLAGS=-DWC_RNG_SEED_CB -DWOLFSSL_TRACK_MEMORY -DWOLFSSL_DEBUG_MEMORY"]}, + {"name": "all-smallstack-cache-trace", "minutes": 2.5, + "configure": ["--enable-smallstackcache", "--enable-smallstack", + "--enable-all", "--enable-debug-trace-errcodes", + "CFLAGS=-DWC_RNG_SEED_CB -DWOLFSSL_TRACK_MEMORY -DWOLFSSL_DEBUG_MEMORY"]}, + {"name": "all-wolfentropy", "minutes": 2.5, + "comment": "smallstackcache coverage here is crucial for the Linux kernel module when targeting a kernel with the randomness patch (linuxkm/patches/) applied. Don't combine wolfEntropy with the full TLS cipher suite test - the implicit wc_InitRng()s in each suite have an enormous CPU footprint.", + "configure": ["--enable-wolfEntropy", "--enable-smallstackcache", + "--enable-smallstack", "--enable-all", + "CFLAGS=-DWC_RNG_SEED_CB -DWOLFSSL_TRACK_MEMORY -DWOLFSSL_DEBUG_MEMORY -DNO_WOLFSSL_CIPHER_SUITE_TEST"]}, + {"name": "all-intelrdseed", "minutes": 2.5, + "configure": ["--enable-intelrdseed", "--enable-smallstackcache", + "--enable-smallstack", "--enable-all", + "CFLAGS=-DWC_RNG_SEED_CB -DWOLFSSL_TRACK_MEMORY -DWOLFSSL_DEBUG_MEMORY -DNO_WOLFSSL_CIPHER_SUITE_TEST"]}, + {"name": "all-amdrand", "minutes": 2.5, + "configure": ["--enable-amdrand", "--enable-smallstackcache", + "--enable-smallstack", "--enable-all", + "CFLAGS=-DWC_RNG_SEED_CB -DWOLFSSL_TRACK_MEMORY -DWOLFSSL_DEBUG_MEMORY -DNO_WOLFSSL_CIPHER_SUITE_TEST"]} + ] + EOF + .github/scripts/parallel-make-check.py \ + --private-dir=certs \ + "$RUNNER_TEMP/trackmemory-configs.json" + + - name: ccache stats + if: always() + run: ccache -s || true + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: trackmemory-logs + path: | + build-*/make-check.log + build-*/test-suite.log + build-*/config.log + if-no-files-found: ignore diff --git a/.github/workflows/wolfCrypt-Wconversion.yml b/.github/workflows/wolfCrypt-Wconversion.yml index 2b0693e155..bbe73f4bac 100644 --- a/.github/workflows/wolfCrypt-Wconversion.yml +++ b/.github/workflows/wolfCrypt-Wconversion.yml @@ -14,40 +14,145 @@ concurrency: # END OF COMMON SECTION jobs: + # All former runner-per-config matrix entries build on one runner via + # .github/scripts/parallel-make-check.py (see os-check.yml for the full + # pattern): each config builds in its own out-of-tree ("VPATH") build + # directory off one checkout/autogen, on a pool of one-per-CPU worker + # threads, longest first. build_library: - strategy: - matrix: - config: [ - # Add new configs here - '--disable-asm --enable-cryptonly --enable-all-crypto --disable-examples --disable-benchmark --disable-crypttests --enable-mlkem --enable-slhdsa --enable-mldsa=yes,small --enable-lms --enable-xmss CPPFLAGS="-DWOLFSSL_MLDSA_ALIGNMENT=0 -DWC_XMSS_FULL_HASH -Wconversion -Warith-conversion -Wenum-conversion -Wfloat-conversion -Wsign-conversion -Wcast-qual"', - '--enable-intelasm --enable-cryptonly --enable-all-crypto --disable-examples --disable-benchmark --disable-crypttests --enable-mlkem --enable-slhdsa=yes,sha2 --enable-mldsa=yes,draft --enable-lms --enable-xmss CPPFLAGS="-DWC_LMS_FULL_HASH -DWOLFSSL_LMS_LARGE_CACHES -Wconversion -Warith-conversion -Wenum-conversion -Wfloat-conversion -Wsign-conversion -Wcast-qual"', - '--enable-smallstack --disable-asm --enable-cryptonly --enable-all-crypto --disable-examples --disable-benchmark --disable-crypttests --enable-mlkem --enable-slhdsa --enable-mldsa=yes,no-ctx --enable-lms=yes,small --enable-xmss CPPFLAGS="-DWOLFSSL_MLDSA_SIGN_SMALL_MEM -DWOLFSSL_MLDSA_VERIFY_SMALL_MEM -DWOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM -DWOLFSSL_XMSS_LARGE_SECRET_KEY -Wconversion -Warith-conversion -Wenum-conversion -Wfloat-conversion -Wsign-conversion -Wcast-qual"', - '--enable-smallstack --enable-intelasm --enable-cryptonly --enable-all-crypto --disable-examples --disable-benchmark --disable-crypttests --enable-mlkem --enable-slhdsa=yes,sha2 --enable-mldsa --enable-lms --enable-xmss CPPFLAGS="-DWOLFSSL_MLDSA_SIGN_SMALL_MEM -DWOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC -DWOLFSSL_WC_LMS_SERIALIZE_STATE -Wconversion -Warith-conversion -Wenum-conversion -Wfloat-conversion -Wsign-conversion -Wcast-qual"', - '--enable-cryptonly --enable-all-crypto --disable-examples --disable-benchmark --disable-crypttests --enable-mlkem --enable-slhdsa=yes,sha2 --enable-mldsa --enable-lms --enable-xmss CPPFLAGS="-DWOLFSSL_MLDSA_SIGN_SMALL_MEM -DWOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC_A -DWOLFSSL_WC_XMSS_NO_SHA512 -DWOLFSSL_LMS_NO_SIG_CACHE -Wconversion -Warith-conversion -Wenum-conversion -Wfloat-conversion -Wsign-conversion -DNO_INT128 -Wcast-qual"', - '--enable-cryptonly --enable-all-crypto --disable-examples --disable-benchmark --disable-crypttests --enable-mlkem --enable-slhdsa=yes,sha2 --enable-mldsa=yes,verify-only --enable-lms=yes,small,sha256-192,shake256 --enable-xmss=yes,verify-only CPPFLAGS="-DWOLFSSL_MLDSA_VERIFY_SMALL_MEM -DWOLFSSL_MLDSA_VERIFY_NO_MALLOC -DWOLFSSL_MLDSA_SMALL_MEM_POLY64 -DWOLFSSL_WC_XMSS_NO_SHAKE128 -DWOLFSSL_WC_XMSS_NO_SHAKE256 -Wdeclaration-after-statement -Wconversion -Warith-conversion -Wenum-conversion -Wfloat-conversion -Wsign-conversion -Wcast-qual" --enable-32bit CFLAGS=-m32', - '--enable-cryptonly --enable-all-crypto --disable-examples --disable-benchmark --disable-crypttests --enable-mlkem=yes,small --enable-slhdsa=yes,small --enable-mldsa --enable-lms --enable-xmss=yes,small CPPFLAGS="-DWC_MLDSA_CACHE_MATRIX_A -DWOLFSSL_LMS_NO_SIGN_SMOOTHING -Wconversion -Warith-conversion -Wenum-conversion -Wfloat-conversion -Wsign-conversion -Wcast-qual -DNO_INT128"', - '--enable-cryptonly --enable-all-crypto --disable-examples --disable-benchmark --disable-crypttests --enable-mlkem=yes,no-large-code --enable-slhdsa=yes,small-mem --enable-mldsa --enable-lms=yes,sha256-192,shake256 --enable-xmss CPPFLAGS="-DWOLFSSL_MLDSA_NO_LARGE_CODE -Wconversion -Warith-conversion -Wenum-conversion -Wfloat-conversion -Wsign-conversion -Wcast-qual -DNO_INT128"', - '--enable-smallstack --enable-cryptonly --enable-all-crypto --disable-examples --disable-benchmark --disable-crypttests --enable-mlkem --enable-slhdsa --enable-mldsa --enable-lms=yes,verify-only --enable-xmss CPPFLAGS="-DWC_MLDSA_CACHE_PRIV_VECTORS -DWC_MLDSA_CACHE_PUB_VECTORS -DWOLFSSL_MLDSA_DYNAMIC_KEYS -Wconversion -Warith-conversion -Wenum-conversion -Wfloat-conversion -Wsign-conversion -Wcast-qual -DNO_INT128"', - '--disable-intelasm --enable-cryptonly --enable-all-crypto --disable-examples --disable-benchmark --disable-crypttests --enable-mlkem --enable-slhdsa=yes,verify-only --enable-mldsa --enable-lms --enable-xmss CPPFLAGS="-DWOLFSSL_MLKEM_ENCAPSULATE_SMALL_MEM -DWOLFSSL_MLKEM_MAKEKEY_SMALL_MEM -DWOLFSSL_MLDSA_NO_ASN1 -DWOLFSSL_MLDSA_ALIGNMENT=0 -Wdeclaration-after-statement -Wconversion -Warith-conversion -Wenum-conversion -Wfloat-conversion -Wsign-conversion -Wcast-qual" --enable-32bit CFLAGS=-m32', - '--disable-intelasm --enable-cryptonly --enable-all-crypto --disable-examples --disable-benchmark --disable-crypttests --enable-mlkem=yes,small --enable-slhdsa --enable-lms --enable-xmss CPPFLAGS="-DWOLFSSL_MLKEM_ENCAPSULATE_SMALL_MEM -DWOLFSSL_MLKEM_MAKEKEY_SMALL_MEM -Wconversion -Warith-conversion -Wenum-conversion -Wfloat-conversion -Wsign-conversion -Wcast-qual -DNO_INT128"', - ] name: build library if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} runs-on: ubuntu-24.04 - # This should be a safe limit for the tests to run. - timeout-minutes: 10 + # Generous for a cold ccache; warm reruns finish in a fraction. + timeout-minutes: 15 steps: - uses: actions/checkout@v4 name: Checkout wolfSSL - - name: Install multilib + - name: Install dependencies uses: ./.github/actions/install-apt-deps with: - packages: gcc-multilib + packages: autoconf automake libtool build-essential gcc-multilib - - name: Build wolfCrypt with extra type conversion warnings + # ccache via the cross-platform composite; the script passes the + # compiler to configure as CC="ccache gcc" (or a per-config "cc"). + - name: Set up ccache + uses: ./.github/actions/ccache-setup + with: + workflow-id: wconversion + max-size: 300M + + # The JSON list below is the former runner-per-config matrix. These + # are compile-only warning checks ("check": false): the -Wconversion + # family must come out clean, nothing is executed. + - name: Build all configs (parallel, out-of-tree) run: | - ./autogen.sh || $(exit 2) - echo "running ./configure ${{ matrix.config }}" - ./configure ${{ matrix.config }} || $(exit 3) - make -j 4 || $(exit 4) + cat > "$RUNNER_TEMP/wconversion-configs.json" <<'EOF' + [ + {"name": "noasm-mldsa-align0", "minutes": 1, + "configure": ["--disable-asm", "--enable-cryptonly", + "--enable-all-crypto", "--disable-examples", "--disable-benchmark", + "--disable-crypttests", "--enable-mlkem", "--enable-slhdsa", + "--enable-mldsa=yes,small", "--enable-lms", "--enable-xmss", + "CPPFLAGS=-DWOLFSSL_MLDSA_ALIGNMENT=0 -DWC_XMSS_FULL_HASH -Wconversion -Warith-conversion -Wenum-conversion -Wfloat-conversion -Wsign-conversion -Wcast-qual"], + "check": false}, + {"name": "intelasm-lms-full-hash", "minutes": 1, + "configure": ["--enable-intelasm", "--enable-cryptonly", + "--enable-all-crypto", "--disable-examples", "--disable-benchmark", + "--disable-crypttests", "--enable-mlkem", + "--enable-slhdsa=yes,sha2", "--enable-mldsa=yes,draft", + "--enable-lms", "--enable-xmss", + "CPPFLAGS=-DWC_LMS_FULL_HASH -DWOLFSSL_LMS_LARGE_CACHES -Wconversion -Warith-conversion -Wenum-conversion -Wfloat-conversion -Wsign-conversion -Wcast-qual"], + "check": false}, + {"name": "smallstack-noasm-small-mem", "minutes": 1, + "configure": ["--enable-smallstack", "--disable-asm", + "--enable-cryptonly", "--enable-all-crypto", "--disable-examples", + "--disable-benchmark", "--disable-crypttests", "--enable-mlkem", + "--enable-slhdsa", "--enable-mldsa=yes,no-ctx", + "--enable-lms=yes,small", "--enable-xmss", + "CPPFLAGS=-DWOLFSSL_MLDSA_SIGN_SMALL_MEM -DWOLFSSL_MLDSA_VERIFY_SMALL_MEM -DWOLFSSL_MLDSA_MAKE_KEY_SMALL_MEM -DWOLFSSL_XMSS_LARGE_SECRET_KEY -Wconversion -Warith-conversion -Wenum-conversion -Wfloat-conversion -Wsign-conversion -Wcast-qual"], + "check": false}, + {"name": "smallstack-intelasm-precalc", "minutes": 1, + "configure": ["--enable-smallstack", "--enable-intelasm", + "--enable-cryptonly", "--enable-all-crypto", "--disable-examples", + "--disable-benchmark", "--disable-crypttests", "--enable-mlkem", + "--enable-slhdsa=yes,sha2", "--enable-mldsa", "--enable-lms", + "--enable-xmss", + "CPPFLAGS=-DWOLFSSL_MLDSA_SIGN_SMALL_MEM -DWOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC -DWOLFSSL_WC_LMS_SERIALIZE_STATE -Wconversion -Warith-conversion -Wenum-conversion -Wfloat-conversion -Wsign-conversion -Wcast-qual"], + "check": false}, + {"name": "precalc-a-no-int128", "minutes": 1, + "configure": ["--enable-cryptonly", "--enable-all-crypto", + "--disable-examples", "--disable-benchmark", + "--disable-crypttests", "--enable-mlkem", + "--enable-slhdsa=yes,sha2", "--enable-mldsa", "--enable-lms", + "--enable-xmss", + "CPPFLAGS=-DWOLFSSL_MLDSA_SIGN_SMALL_MEM -DWOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC_A -DWOLFSSL_WC_XMSS_NO_SHA512 -DWOLFSSL_LMS_NO_SIG_CACHE -Wconversion -Warith-conversion -Wenum-conversion -Wfloat-conversion -Wsign-conversion -DNO_INT128 -Wcast-qual"], + "check": false}, + {"name": "cache-matrix-no-smoothing", "minutes": 1, + "configure": ["--enable-cryptonly", "--enable-all-crypto", + "--disable-examples", "--disable-benchmark", + "--disable-crypttests", "--enable-mlkem=yes,small", + "--enable-slhdsa=yes,small", "--enable-mldsa", "--enable-lms", + "--enable-xmss=yes,small", + "CPPFLAGS=-DWC_MLDSA_CACHE_MATRIX_A -DWOLFSSL_LMS_NO_SIGN_SMOOTHING -Wconversion -Warith-conversion -Wenum-conversion -Wfloat-conversion -Wsign-conversion -Wcast-qual -DNO_INT128"], + "check": false}, + {"name": "no-large-code-lms192", "minutes": 1, + "configure": ["--enable-cryptonly", "--enable-all-crypto", + "--disable-examples", "--disable-benchmark", + "--disable-crypttests", "--enable-mlkem=yes,no-large-code", + "--enable-slhdsa=yes,small-mem", "--enable-mldsa", + "--enable-lms=yes,sha256-192,shake256", "--enable-xmss", + "CPPFLAGS=-DWOLFSSL_MLDSA_NO_LARGE_CODE -Wconversion -Warith-conversion -Wenum-conversion -Wfloat-conversion -Wsign-conversion -Wcast-qual -DNO_INT128"], + "check": false}, + {"name": "smallstack-mldsa-dynamic", "minutes": 1, + "configure": ["--enable-smallstack", "--enable-cryptonly", + "--enable-all-crypto", "--disable-examples", "--disable-benchmark", + "--disable-crypttests", "--enable-mlkem", "--enable-slhdsa", + "--enable-mldsa", "--enable-lms=yes,verify-only", "--enable-xmss", + "CPPFLAGS=-DWC_MLDSA_CACHE_PRIV_VECTORS -DWC_MLDSA_CACHE_PUB_VECTORS -DWOLFSSL_MLDSA_DYNAMIC_KEYS -Wconversion -Warith-conversion -Wenum-conversion -Wfloat-conversion -Wsign-conversion -Wcast-qual -DNO_INT128"], + "check": false}, + {"name": "mlkem-small-no-int128", "minutes": 1, + "configure": ["--disable-intelasm", "--enable-cryptonly", + "--enable-all-crypto", "--disable-examples", "--disable-benchmark", + "--disable-crypttests", "--enable-mlkem=yes,small", + "--enable-slhdsa", "--enable-lms", "--enable-xmss", + "CPPFLAGS=-DWOLFSSL_MLKEM_ENCAPSULATE_SMALL_MEM -DWOLFSSL_MLKEM_MAKEKEY_SMALL_MEM -Wconversion -Warith-conversion -Wenum-conversion -Wfloat-conversion -Wsign-conversion -Wcast-qual -DNO_INT128"], + "check": false}, + {"name": "verify-only-m32", "minutes": 0.8, + "configure": ["--enable-cryptonly", "--enable-all-crypto", + "--disable-examples", "--disable-benchmark", + "--disable-crypttests", "--enable-mlkem", + "--enable-slhdsa=yes,sha2", "--enable-mldsa=yes,verify-only", + "--enable-lms=yes,small,sha256-192,shake256", + "--enable-xmss=yes,verify-only", + "CPPFLAGS=-DWOLFSSL_MLDSA_VERIFY_SMALL_MEM -DWOLFSSL_MLDSA_VERIFY_NO_MALLOC -DWOLFSSL_MLDSA_SMALL_MEM_POLY64 -DWOLFSSL_WC_XMSS_NO_SHAKE128 -DWOLFSSL_WC_XMSS_NO_SHAKE256 -Wdeclaration-after-statement -Wconversion -Warith-conversion -Wenum-conversion -Wfloat-conversion -Wsign-conversion -Wcast-qual", + "--enable-32bit", "CFLAGS=-m32"], + "check": false}, + {"name": "m32-mlkem-small-mem", "minutes": 0.8, + "configure": ["--disable-intelasm", "--enable-cryptonly", + "--enable-all-crypto", "--disable-examples", "--disable-benchmark", + "--disable-crypttests", "--enable-mlkem", + "--enable-slhdsa=yes,verify-only", "--enable-mldsa", + "--enable-lms", "--enable-xmss", + "CPPFLAGS=-DWOLFSSL_MLKEM_ENCAPSULATE_SMALL_MEM -DWOLFSSL_MLKEM_MAKEKEY_SMALL_MEM -DWOLFSSL_MLDSA_NO_ASN1 -DWOLFSSL_MLDSA_ALIGNMENT=0 -Wdeclaration-after-statement -Wconversion -Warith-conversion -Wenum-conversion -Wfloat-conversion -Wsign-conversion -Wcast-qual", + "--enable-32bit", "CFLAGS=-m32"], + "check": false} + ] + EOF + .github/scripts/parallel-make-check.py \ + "$RUNNER_TEMP/wconversion-configs.json" + + - name: ccache stats + if: always() + run: ccache -s || true + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: wconversion-logs + path: | + build-*/make-check.log + build-*/test-suite.log + build-*/config.log + if-no-files-found: ignore diff --git a/.github/workflows/wolfsm.yml b/.github/workflows/wolfsm.yml index 36748af3a9..dbe649426f 100644 --- a/.github/workflows/wolfsm.yml +++ b/.github/workflows/wolfsm.yml @@ -14,22 +14,19 @@ concurrency: # END OF COMMON SECTION jobs: + # All former runner-per-config matrix entries build on one runner via + # .github/scripts/parallel-make-check.py (see os-check.yml for the full + # pattern): each config in its own out-of-tree ("VPATH") build directory + # off one checkout/autogen, checks on a pool of one-per-CPU worker + # threads, longest first. bubblewrap gives every test script its own + # network namespace so concurrent checks cannot collide on TCP/UDP ports + # (do not set AM_BWRAPPED here - that would disable it). make_check: - strategy: - fail-fast: false - matrix: - config: [ - # Core SM TLS cipher suites - '--enable-sm2 --enable-sm3 --enable-sm4-gcm --enable-sm4-ccm --enable-sha3', - # All SM4 modes - '--enable-sm2 --enable-sm3 --enable-sm4-ecb --enable-sm4-cbc --enable-sm4-ctr --enable-sm4-gcm --enable-sm4-ccm --enable-sha3', - # SM + all features integration test - '--enable-all --enable-sm2 --enable-sm3 --enable-sm4-ecb --enable-sm4-cbc --enable-sm4-ctr --enable-sm4-gcm --enable-sm4-ccm', - ] name: make check if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} runs-on: ubuntu-24.04 - timeout-minutes: 10 + # Generous for a cold ccache; warm reruns finish in a fraction. + timeout-minutes: 15 steps: - uses: actions/checkout@v4 name: Checkout wolfSSL @@ -41,25 +38,70 @@ jobs: path: wolfsm fetch-depth: 1 + # Patches the wolfSSL source tree in place (adds the SM sources); + # must run before the script's autogen/configure. - name: Install wolfsm working-directory: wolfsm run: ./install.sh $GITHUB_WORKSPACE - - name: Test wolfSSL with wolfSM - run: | - ./autogen.sh - ./configure ${{ matrix.config }} - make -j - make check + - name: Install dependencies + uses: ./.github/actions/install-apt-deps + with: + packages: autoconf automake libtool build-essential bubblewrap - - name: Print errors - if: ${{ failure() }} + # ccache via the cross-platform composite; the script passes the + # compiler to configure as CC="ccache gcc" (or a per-config "cc"). + - name: Set up ccache + uses: ./.github/actions/ccache-setup + with: + workflow-id: wolfsm + max-size: 200M + + # Ubuntu 24.04 can restrict unprivileged user namespaces via AppArmor, + # which would stop the test scripts from re-execing under + # bwrap --unshare-net (their port-isolation mechanism). + - name: Allow unprivileged user namespaces (for bwrap) + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true + + # The JSON list below is the former runner-per-config matrix; add new + # configs as new entries (a "comment" key is allowed for notes). + # "minutes" is the expected duration driving longest-first scheduling: + # take it from the Minutes column of a previous run's step summary, or + # omit it for a new config (defaults to 1) and refresh later. The list + # is kept sorted by minutes for readability, but the schedule sorts by + # the values, not list order. + - name: Build and make check all configs (parallel, out-of-tree) run: | - for file in scripts/*.log - do - if [ -f "$file" ]; then - echo "${file}:" - cat "$file" - echo "========================================================================" - fi - done + cat > "$RUNNER_TEMP/wolfsm-configs.json" <<'EOF' + [ + {"name": "all-sm", "minutes": 3, + "configure": ["--enable-all", "--enable-sm2", "--enable-sm3", + "--enable-sm4-ecb", "--enable-sm4-cbc", "--enable-sm4-ctr", + "--enable-sm4-gcm", "--enable-sm4-ccm"]}, + {"name": "sm-tls-suites", "minutes": 1.5, + "configure": ["--enable-sm2", "--enable-sm3", "--enable-sm4-gcm", + "--enable-sm4-ccm", "--enable-sha3"]}, + {"name": "sm4-all-modes", "minutes": 1.5, + "configure": ["--enable-sm2", "--enable-sm3", "--enable-sm4-ecb", + "--enable-sm4-cbc", "--enable-sm4-ctr", "--enable-sm4-gcm", + "--enable-sm4-ccm", "--enable-sha3"]} + ] + EOF + .github/scripts/parallel-make-check.py \ + --private-dir=certs \ + "$RUNNER_TEMP/wolfsm-configs.json" + + - name: ccache stats + if: always() + run: ccache -s || true + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: wolfsm-logs + path: | + build-*/make-check.log + build-*/test-suite.log + build-*/config.log + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index 93388d03bf..693d9d5a84 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ build-aux/ rpm/spec *.rpm stamp-h +wolfssl-test-data.stamp cyassl/options.h wolfssl/options.h .build_params @@ -207,6 +208,8 @@ NTRU_algorithm/ NTRU/ build-test/ build/ +# Out-of-tree build dirs created by .github/scripts/parallel-make-check.py +/build-*/ cyassl.xcodeproj/ cyassl*rc* autoscan.log diff --git a/Makefile.am b/Makefile.am index 2d70ba2dd6..6add8d5904 100644 --- a/Makefile.am +++ b/Makefile.am @@ -134,6 +134,12 @@ CLEANFILES+= ecc-key.der \ pkcs7signedEncryptedCompressedFirmwarePkgData_RSA_SHA256.der \ pkcs7signedEncryptedCompressedFirmwarePkgData_RSA_SHA256_noattr.der \ tests/test-log-dump-to-file.txt \ + tests/bio_write_test.txt \ + tests/cert_cache.tmp \ + certeccrsa.der \ + certeccrsa.pem \ + ecc-key.pem \ + test-write-dhparams.pem \ MyKeyLog.txt exampledir = $(docdir)/example @@ -274,6 +280,68 @@ TESTS += $(check_PROGRAMS) check_SCRIPTS+= $(dist_noinst_SCRIPTS) TESTS += $(check_SCRIPTS) +############################################################################## +# Out-of-tree ("VPATH") build support for "make check". +# +# The test programs (testsuite, tests/unit.test) and the example client and +# server binaries, as well as the shell-script tests under scripts/, locate +# their read-only inputs using paths relative to the working directory: +# certificates under certs/, TLS test-configuration files under tests/, sniffer +# captures and helpers under scripts/, and the top-level "input"/"quit" files. +# ChangeToWolfRoot() (wolfssl/test.h) walks up from the working directory +# looking for certs/dh2048.pem to anchor these relative paths. +# +# For an in-tree build that data is already present in the build directory, but +# for an out-of-tree build it exists only in the source tree. Symlink it into +# the build tree so the tests can find it. This is driven by a stamp file in +# BUILT_SOURCES so it runs once, before anything else, for "make", "make all" +# and "make check". It is a no-op for in-tree builds. +############################################################################## +BUILT_SOURCES += wolfssl-test-data.stamp + +wolfssl-test-data.stamp: + $(AM_V_at)if test "$(abs_top_srcdir)" != "$(abs_top_builddir)"; then \ + $(MKDIR_P) tests scripts examples; \ + for f in certs input quit; do \ + rm -f "$$f"; \ + $(LN_S) "$(abs_top_srcdir)/$$f" "$$f"; \ + done; \ + rm -f examples/crypto_policies; \ + $(LN_S) "$(abs_top_srcdir)/examples/crypto_policies" \ + examples/crypto_policies; \ + for f in "$(abs_top_srcdir)"/tests/*.conf \ + "$(abs_top_srcdir)"/tests/*.cnf \ + "$(abs_top_srcdir)"/tests/TXT_DB.txt; do \ + test -e "$$f" || continue; \ + b=`basename "$$f"`; \ + rm -f "tests/$$b"; \ + $(LN_S) "$$f" "tests/$$b"; \ + done; \ + for f in "$(abs_top_srcdir)"/scripts/*.pcap \ + "$(abs_top_srcdir)"/scripts/*.out \ + "$(abs_top_srcdir)"/scripts/*.sslkeylog \ + "$(abs_top_srcdir)"/scripts/multi-msg-record.py; do \ + test -e "$$f" || continue; \ + b=`basename "$$f"`; \ + rm -f "scripts/$$b"; \ + $(LN_S) "$$f" "scripts/$$b"; \ + done; \ + fi + $(AM_V_at)touch $@ + +DISTCLEANFILES += wolfssl-test-data.stamp + +# Remove the symlinks created for out-of-tree builds. Gated on +# srcdir != builddir so an in-tree build never touches the real source files. +distclean-local: + $(AM_V_at)if test "$(abs_top_srcdir)" != "$(abs_top_builddir)"; then \ + rm -f certs input quit; \ + rm -f tests/*.conf tests/*.cnf tests/TXT_DB.txt; \ + rm -f scripts/*.pcap scripts/*.out scripts/*.sslkeylog \ + scripts/multi-msg-record.py; \ + rm -f examples/crypto_policies; \ + fi + test: check @BUILD_EXAMPLE_SERVERS_TRUE@tests/unit.log: testsuite/testsuite.log @BUILD_EXAMPLE_SERVERS_TRUE@scripts/unit.log: testsuite/testsuite.log diff --git a/configure.ac b/configure.ac index d12f09579a..bdf5d3df29 100644 --- a/configure.ac +++ b/configure.ac @@ -10768,9 +10768,6 @@ then if test "$ENABLED_CRYPTOCB" != "yes" && test "$enable_usersettings" != "yes"; then AC_MSG_ERROR([--enable-swdev requires --enable-cryptocb (or --enable-usersettings with WOLF_CRYPTO_CB defined in user_settings.h)]) fi - if test "x$srcdir" != "x."; then - AC_MSG_ERROR([--enable-swdev currently supports in-tree builds only]) - fi AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_SWDEV -DWOLF_CRYPTO_CB_FIND" fi @@ -12659,7 +12656,7 @@ echo "" >> $OPTION_FILE if test "$ENABLED_DEBUG_TRACE_ERRCODES" != "no" then - support/gen-debug-trace-error-codes.sh || AC_MSG_ERROR([Header generation for debug-trace-errcodes failed.]) + "$srcdir"/support/gen-debug-trace-error-codes.sh "$srcdir" || AC_MSG_ERROR([Header generation for debug-trace-errcodes failed.]) fi if test "$ENABLED_OPENSSLEXTRA" = "yes" && test "$ENABLED_LINUXKM" = "no" diff --git a/scripts/multi-msg-record.py b/scripts/multi-msg-record.py index ae035d5605..8145f62f03 100755 --- a/scripts/multi-msg-record.py +++ b/scripts/multi-msg-record.py @@ -40,7 +40,20 @@ import time import types SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -WOLFSSL_DIR = os.path.dirname(SCRIPT_DIR) + + +def _find_wolfssl_dir(): + # Under `make check` the working directory is the build tree, which is + # where the client binary lives and which differs from the tree + # containing this script in an out-of-tree (VPATH) build. Fall back to + # the script's parent for direct invocation from the source tree. + for root in (os.getcwd(), os.path.dirname(SCRIPT_DIR)): + if os.path.isfile(os.path.join(root, "examples", "client", "client")): + return root + return os.path.dirname(SCRIPT_DIR) + + +WOLFSSL_DIR = _find_wolfssl_dir() WOLF_CLIENT = os.path.join(WOLFSSL_DIR, "examples", "client", "client") CERT_DIR = os.path.join(WOLFSSL_DIR, "certs") diff --git a/support/gen-debug-trace-error-codes.sh b/support/gen-debug-trace-error-codes.sh index 540a95273b..4b27b072e3 100755 --- a/support/gen-debug-trace-error-codes.sh +++ b/support/gen-debug-trace-error-codes.sh @@ -1,5 +1,14 @@ #!/bin/sh +# The input error-code headers are read-only sources. For out-of-tree (VPATH) +# builds they live in the source tree, which may differ from the build tree, so +# read them from $srcdir (first argument, defaulting to the current directory). +# The generated headers are written relative to the current directory, i.e. +# into the build tree. +srcdir="${1:-.}" + +mkdir -p wolfssl + awk ' BEGIN { print("/* automatically generated, do not edit */") > "wolfssl/debug-trace-error-codes.h"; @@ -39,4 +48,4 @@ END { print("") >> "wolfssl/debug-untrace-error-codes.h"; print("#endif /* WOLFSSL_DEBUG_TRACE_ERROR_CODES_H */") >> "wolfssl/debug-untrace-error-codes.h"; -}' wolfssl/wolfcrypt/error-crypt.h wolfssl/error-ssl.h +}' "$srcdir/wolfssl/wolfcrypt/error-crypt.h" "$srcdir/wolfssl/error-ssl.h" diff --git a/tests/swdev/Makefile b/tests/swdev/Makefile index 971beb6e1e..c771604c0d 100644 --- a/tests/swdev/Makefile +++ b/tests/swdev/Makefile @@ -5,6 +5,10 @@ SRCDIR ?= $(CURDIR) WOLFROOT ?= $(abspath $(SRCDIR)/../..) +# Parent build tree; differs from WOLFROOT in out-of-tree (VPATH) builds, +# where configure-generated headers (wolfssl/options.h, wolfssl/version.h) +# live in the build tree. +WOLFBUILD ?= $(WOLFROOT) BUILDDIR ?= $(SRCDIR)/build CC ?= cc @@ -37,6 +41,7 @@ CPPFLAGS_SWDEV = \ -DWOLFSSL_USER_SETTINGS \ -I$(SRCDIR) \ $(PARENT_CPPFLAGS) \ + -I$(WOLFBUILD) \ -I$(WOLFROOT) \ -I$(WOLFROOT)/wolfssl @@ -62,7 +67,7 @@ endif # Choose between options.h and parent user_settings.h ifeq ($(PARENT_USER_SETTINGS_H),) -SWDEV_CFG_PREREQ = $(WOLFROOT)/wolfssl/options.h +SWDEV_CFG_PREREQ = $(WOLFBUILD)/wolfssl/options.h else SWDEV_CFG_PREREQ = $(PARENT_USER_SETTINGS_H) endif diff --git a/tests/swdev/README.md b/tests/swdev/README.md index 3fc8f24e36..eedbe1e5a1 100644 --- a/tests/swdev/README.md +++ b/tests/swdev/README.md @@ -130,10 +130,11 @@ Notes: with `WOLF_CRYPTO_CB` defined in the user's `user_settings.h`. - `--enable-swdev` defines `WOLFSSL_SWDEV` and `WOLF_CRYPTO_CB_FIND` automatically; see `configure.ac`. -- `--enable-swdev` currently supports **in-tree builds only**. - Out-of-tree (VPATH) builds fail at configure time. swdev is built - from `wolfcrypt/test/include.am` and inherits `PARENT_SRCS`, - `PARENT_BUILD_CFLAGS`, etc., from the parent build. +- swdev is built from `wolfcrypt/test/include.am` and inherits + `PARENT_SRCS`, `PARENT_BUILD_CFLAGS`, etc., from the parent build. + Out-of-tree (VPATH) builds work: `WOLFBUILD` points the sub-make at + the build tree for the configure-generated headers + (`wolfssl/options.h`, `wolfssl/version.h`). For the full CI matrix that exercises each `_ONLY_*` macro, see `.github/workflows/cryptocb-only.yml`. diff --git a/wolfcrypt/test/include.am b/wolfcrypt/test/include.am index 359bca42e8..c9d5943ecc 100644 --- a/wolfcrypt/test/include.am +++ b/wolfcrypt/test/include.am @@ -24,13 +24,10 @@ endif if BUILD_SWDEV # delegates to tests/swdev/Makefile $(top_builddir)/tests/swdev/build/swdev.o: $(top_builddir)/wolfssl/options.h FORCE - @test "$(abs_top_srcdir)" = "$(abs_top_builddir)" || { \ - echo "error: --enable-swdev currently supports in-tree builds only"; \ - exit 1; \ - } $(MAKE) -C $(top_srcdir)/tests/swdev \ SRCDIR=$(abs_top_srcdir)/tests/swdev \ WOLFROOT=$(abs_top_srcdir) \ + WOLFBUILD=$(abs_top_builddir) \ $(if $(CC),CC='$(CC)') \ $(if $(CCAS),CCAS='$(CCAS)') \ $(if $(LD),LD='$(LD)') \ diff --git a/wolfssl/include.am b/wolfssl/include.am index 5c20caf4a8..0da35486d3 100644 --- a/wolfssl/include.am +++ b/wolfssl/include.am @@ -38,4 +38,4 @@ nobase_include_HEADERS+= wolfssl/options.h endif wolfssl/debug-trace-error-codes.h wolfssl/debug-untrace-error-codes.h: wolfssl/wolfcrypt/error-crypt.h wolfssl/error-ssl.h - @support/gen-debug-trace-error-codes.sh + @$(SHELL) $(top_srcdir)/support/gen-debug-trace-error-codes.sh $(top_srcdir) From a62884599b86292006c34d43b4d85b2cf9eb1f4b Mon Sep 17 00:00:00 2001 From: Juliusz Sosinowicz Date: Thu, 11 Jun 2026 19:50:33 +0000 Subject: [PATCH 2/9] CI review fixes: JSON validation, log volume, rm -rf, flag spelling Address the Copilot review: - parallel-make-check.py: validate "configure" (list of strings) and cflags/ldflags (strings) so a malformed entry fails the load instead of exploding a string into per-character configure arguments; print a single line for passing configs instead of dumping their full make-check.log into the CI log (failure dumps unchanged; the logs remain in build-/ for the failure artifacts). - Makefile.am: use rm -rf for the certs/input/quit setup and distclean cleanup. A --private-dir run replaces the certs symlink with a private directory copy that rm -f cannot remove (verified: make distclean in a build dir with a privatized certs/ now succeeds and removes it). - psk.yml, disable-pk-algs.yml: normalize the single-dash tokens (-disable-rsa, -disable-ecc, -disable-aescbc, -enable-cryptonly) carried verbatim from the old matrices to the canonical double-dash form. No coverage change: configure honors single-dash spellings (verified -disable-rsa sets NO_RSA with no unrecognized-option warning), so these were always in effect; both touched configs re-validated end-to-end. The --cc default stays "ccache gcc": ccache resolves the compiler through its own masquerade symlinks (verified: no recursion and normal cache hits with /usr/lib/ccache prepended to PATH), and the explicit CC= also covers jobs that use ccache without the PATH masquerade. --- .github/scripts/parallel-make-check.py | 18 +++++++++++++++--- .github/workflows/disable-pk-algs.yml | 2 +- .github/workflows/psk.yml | 10 +++++----- Makefile.am | 8 ++++++-- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/.github/scripts/parallel-make-check.py b/.github/scripts/parallel-make-check.py index 4f4cf2ce82..57d0b61d3a 100755 --- a/.github/scripts/parallel-make-check.py +++ b/.github/scripts/parallel-make-check.py @@ -142,6 +142,14 @@ def load_configs(opts, error): f"directory suffix: {entry!r}") if any(cfg.name == name for cfg in configs): error(f"{opts.json}: duplicate config name {name!r}") + configure = entry.get("configure", []) + if not (isinstance(configure, list) + and all(isinstance(a, str) for a in configure)): + error(f"{opts.json}: \"configure\" must be a list of argument " + f"strings in {name!r}") + for key in ("cflags", "ldflags"): + if not isinstance(entry.get(key, ""), str): + error(f"{opts.json}: \"{key}\" must be a string in {name!r}") minutes = entry.get("minutes", 1.0) if isinstance(minutes, bool) or not isinstance(minutes, (int, float)) \ or minutes < 0: @@ -165,7 +173,7 @@ def load_configs(opts, error): for cmd in cmds)): error(f"{opts.json}: \"{key}\" must be a list of argv lists " f"in {name!r}") - configs.append(Config(name, list(entry.get("configure", [])), cc, + configs.append(Config(name, list(configure), cc, entry.get("cflags", opts.cflags), entry.get("ldflags", opts.ldflags), float(minutes), user_settings, check, @@ -273,9 +281,13 @@ def run_config(cfg, opts): if failed == "aborted": print(f"{cfg.name}: aborted (fail-fast) [{minutes:.1f} min]") sys.stdout.flush() + elif not failed: + # One line per passing config; the full logs would bloat the CI + # log (they stay in build-/make-check.log). + print(f"{cfg.name}: pass [{minutes:.1f} min]") + sys.stdout.flush() else: - verdict = f"FAIL ({failed})" if failed else "pass" - dump(f"{cfg.name}: {verdict} [{minutes:.1f} min]", log) + dump(f"{cfg.name}: FAIL ({failed}) [{minutes:.1f} min]", log) if failed == "configure": dump(f"{cfg.name}: config.log", bdir / "config.log") elif failed == "make check": diff --git a/.github/workflows/disable-pk-algs.yml b/.github/workflows/disable-pk-algs.yml index 1517e9ff0b..fa7cdaa210 100644 --- a/.github/workflows/disable-pk-algs.yml +++ b/.github/workflows/disable-pk-algs.yml @@ -94,7 +94,7 @@ jobs: "--disable-curve25519", "--disable-ed25519", "--disable-curve448", "--disable-ed448", "--enable-curve448", "--enable-ed448"]}, {"name": "cryptonly-rsa", "minutes": 0.8, - "configure": ["-enable-cryptonly", "--disable-rsa", "--disable-dh", + "configure": ["--enable-cryptonly", "--disable-rsa", "--disable-dh", "--disable-ecc", "--disable-curve25519", "--disable-ed25519", "--disable-curve448", "--disable-ed448", "--enable-rsa"]}, {"name": "cryptonly-dh", "minutes": 0.8, diff --git a/.github/workflows/psk.yml b/.github/workflows/psk.yml index 2bb512cb5b..6024db1b78 100644 --- a/.github/workflows/psk.yml +++ b/.github/workflows/psk.yml @@ -73,23 +73,23 @@ jobs: "--disable-ecc", "--disable-dh"]}, {"name": "static-psk-lowresource-tls12", "minutes": 0.8, "configure": ["--disable-oldtls", "--disable-tls13", "--enable-psk", - "-disable-rsa", "--disable-dh", "-disable-ecc", "--disable-asn", + "--disable-rsa", "--disable-dh", "--disable-ecc", "--disable-asn", "C_EXTRA_FLAGS=-DWOLFSSL_STATIC_PSK", "--enable-lowresource", "--enable-singlethreaded", "--disable-asm", "--disable-errorstrings", "--disable-pkcs12", "--disable-sha3", "--disable-sha224", "--disable-sha384", "--disable-sha512", - "--disable-sha", "--disable-md5", "-disable-aescbc", + "--disable-sha", "--disable-md5", "--disable-aescbc", "--disable-chacha", "--disable-poly1305", "--disable-coding", "--disable-sp-math-all", "--disable-mlkem"]}, {"name": "static-psk-lowresource-tls13", "minutes": 0.8, "configure": ["--disable-oldtls", "--disable-tlsv12", - "--enable-tls13", "--enable-psk", "-disable-rsa", "--disable-dh", - "-disable-ecc", "--disable-asn", + "--enable-tls13", "--enable-psk", "--disable-rsa", "--disable-dh", + "--disable-ecc", "--disable-asn", "C_EXTRA_FLAGS=-DWOLFSSL_STATIC_PSK", "--enable-lowresource", "--enable-singlethreaded", "--disable-asm", "--disable-errorstrings", "--disable-pkcs12", "--disable-sha3", "--disable-sha224", "--disable-sha384", "--disable-sha512", - "--disable-sha", "--disable-md5", "-disable-aescbc", + "--disable-sha", "--disable-md5", "--disable-aescbc", "--disable-chacha", "--disable-poly1305", "--disable-coding", "--disable-sp-math-all", "--disable-mlkem"]} ] diff --git a/Makefile.am b/Makefile.am index 6add8d5904..0215e0600a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -296,6 +296,10 @@ TESTS += $(check_SCRIPTS) # the build tree so the tests can find it. This is driven by a stamp file in # BUILT_SOURCES so it runs once, before anything else, for "make", "make all" # and "make check". It is a no-op for in-tree builds. +# +# The setup and the distclean cleanup use rm -rf: a --private-dir run of +# .github/scripts/parallel-make-check.py replaces the certs symlink with a +# private directory copy, which rm -f would not remove. ############################################################################## BUILT_SOURCES += wolfssl-test-data.stamp @@ -303,7 +307,7 @@ wolfssl-test-data.stamp: $(AM_V_at)if test "$(abs_top_srcdir)" != "$(abs_top_builddir)"; then \ $(MKDIR_P) tests scripts examples; \ for f in certs input quit; do \ - rm -f "$$f"; \ + rm -rf "$$f"; \ $(LN_S) "$(abs_top_srcdir)/$$f" "$$f"; \ done; \ rm -f examples/crypto_policies; \ @@ -335,7 +339,7 @@ DISTCLEANFILES += wolfssl-test-data.stamp # srcdir != builddir so an in-tree build never touches the real source files. distclean-local: $(AM_V_at)if test "$(abs_top_srcdir)" != "$(abs_top_builddir)"; then \ - rm -f certs input quit; \ + rm -rf certs input quit; \ rm -f tests/*.conf tests/*.cnf tests/TXT_DB.txt; \ rm -f scripts/*.pcap scripts/*.out scripts/*.sslkeylog \ scripts/multi-msg-record.py; \ From eb59a12b3662eafdcc2a6fb963749566cd5811b1 Mon Sep 17 00:00:00 2001 From: Juliusz Sosinowicz Date: Thu, 11 Jun 2026 20:28:31 +0000 Subject: [PATCH 3/9] parallel-make-check: close the fail-fast race, contain callable errors Two fixes from the second Copilot review round: A process spawned between abort_others()' live_procs snapshot and its registration escaped the kill sweep, leaving that build/check running to completion after fail-fast had begun. Re-check stop_event right after registering the process and SIGTERM its process group if the abort already started: either the registration happened before the sweep's snapshot (the sweep kills it) or it happened after stop_event was set (the re-check sees it), so the window is closed. Exceptions from callable steps (user_settings staging, private-dir copies) used to escape the worker thread and crash the whole script with no summary. They are now recorded as that config's step failure with the exception written to its make-check.log, e.g. a bad "user_settings" path reports FAIL (stage ) while the other configs keep running; the fail-fast bookkeeping is shared with the nonzero-exit path via record_failure(). --- .github/scripts/parallel-make-check.py | 41 +++++++++++++++++++------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/.github/scripts/parallel-make-check.py b/.github/scripts/parallel-make-check.py index 57d0b61d3a..63d462cb9d 100755 --- a/.github/scripts/parallel-make-check.py +++ b/.github/scripts/parallel-make-check.py @@ -242,13 +242,32 @@ def run_config(cfg, opts): failed = None start = time.monotonic() log = bdir / "make-check.log" + + def record_failure(step): + # Classify a failed step, doing the fail-fast bookkeeping: the + # first failure wins and aborts everyone else; any failure after + # the abort began is reported as aborted instead. + if not opts.fail_fast: + return step + with fail_lock: + label = "aborted" if stop_event.is_set() else step + stop_event.set() + if label != "aborted": + abort_others() + return label + with open(log, "w") as logf: for step, cmd in steps: if opts.fail_fast and stop_event.is_set(): failed = "aborted" break if callable(cmd): - cmd() + try: + cmd() + except Exception as e: # one config's bug, not the run's + print(f"+ {step}: {e!r}", file=logf, flush=True) + failed = record_failure(step) + break continue print(f"+ {' '.join(cmd)}", file=logf, flush=True) # stdin=DEVNULL so a test that reads stdin sees EOF (as in CI) @@ -259,22 +278,22 @@ def run_config(cfg, opts): start_new_session=True) with procs_lock: live_procs.add(proc) + if opts.fail_fast and stop_event.is_set(): + # Close the race with abort_others(): if its sweep ran + # between our stop_event check above and the registration + # just now, this process escaped the sweep - kill it + # ourselves (the wait() below then reaps it quickly). + try: + os.killpg(proc.pid, signal.SIGTERM) + except (ProcessLookupError, PermissionError): + proc.terminate() try: rc = proc.wait() finally: with procs_lock: live_procs.discard(proc) if rc != 0: - if opts.fail_fast: - # The first failure wins; any nonzero exit after the - # abort began was most likely our SIGTERM. - with fail_lock: - failed = "aborted" if stop_event.is_set() else step - stop_event.set() - if failed != "aborted": - abort_others() - else: - failed = step + failed = record_failure(step) break minutes = (time.monotonic() - start) / 60 with print_lock: From 1f6abed28ed0085f84eafb75a85cf87fd09bc572 Mon Sep 17 00:00:00 2001 From: Juliusz Sosinowicz Date: Thu, 11 Jun 2026 20:48:19 +0000 Subject: [PATCH 4/9] CI hardening: stamp under set -e, SIGKILL escalation for fail-fast Third Copilot review round: - Makefile.am: run the test-data stamp recipe body under set -e. A failed symlink mid-loop previously did not fail the compound command (only the last command's status counted), so a partially-populated build tree could be stamped complete. Now any failed setup command aborts the recipe and the stamp is not created. - parallel-make-check.py: fail-fast sent SIGTERM only, so a test that traps or ignores SIGTERM could keep the job alive until the workflow timeout. abort_others() now polls the swept processes and SIGKILLs whatever is still alive after a 10 s grace period, and the post-registration race-window kill escalates the same way (bounded wait, then SIGKILL). Verified with a config running "trap '' TERM; sleep 300": the run completes in ~10 s with the stubborn config reported as aborted and no surviving processes. --- .github/scripts/parallel-make-check.py | 44 +++++++++++++++++++------- Makefile.am | 7 ++-- 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/.github/scripts/parallel-make-check.py b/.github/scripts/parallel-make-check.py index 63d462cb9d..9395470e99 100755 --- a/.github/scripts/parallel-make-check.py +++ b/.github/scripts/parallel-make-check.py @@ -50,8 +50,9 @@ # directory; see --private-dir for the exception. # # The first failing config aborts the others (pending configs are skipped, -# in-flight ones are killed) so CI fails fast; pass --no-fail-fast to run -# everything and report every failure. +# in-flight ones get SIGTERM, then SIGKILL after a 10 s grace period) so CI +# fails fast; pass --no-fail-fast to run everything and report every +# failure. import argparse import json @@ -96,16 +97,33 @@ live_procs = set() procs_lock = threading.Lock() -def abort_others(): - # Every subprocess starts its own session, so killing the process +def kill_group(p, sig): + # Every subprocess starts its own session, so signalling the process # group takes down the whole make/test tree under it. + try: + os.killpg(p.pid, sig) + except (ProcessLookupError, PermissionError): + try: + p.send_signal(sig) + except ProcessLookupError: + pass + + +def abort_others(): with procs_lock: procs = list(live_procs) for p in procs: - try: - os.killpg(p.pid, signal.SIGTERM) - except (ProcessLookupError, PermissionError): - pass + kill_group(p, signal.SIGTERM) + # Bounded escalation: SIGKILL whatever ignored the SIGTERM, so + # fail-fast cannot hang behind a test that traps/ignores SIGTERM. + deadline = time.monotonic() + 10 + while any(p.poll() is None for p in procs): + if time.monotonic() > deadline: + for p in procs: + if p.poll() is None: + kill_group(p, signal.SIGKILL) + break + time.sleep(0.2) def nproc(): @@ -282,11 +300,13 @@ def run_config(cfg, opts): # Close the race with abort_others(): if its sweep ran # between our stop_event check above and the registration # just now, this process escaped the sweep - kill it - # ourselves (the wait() below then reaps it quickly). + # ourselves (the wait() below then reaps it), escalating + # like the sweep does if SIGTERM is ignored. + kill_group(proc, signal.SIGTERM) try: - os.killpg(proc.pid, signal.SIGTERM) - except (ProcessLookupError, PermissionError): - proc.terminate() + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + kill_group(proc, signal.SIGKILL) try: rc = proc.wait() finally: diff --git a/Makefile.am b/Makefile.am index 0215e0600a..4f3f8fce0c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -299,12 +299,15 @@ TESTS += $(check_SCRIPTS) # # The setup and the distclean cleanup use rm -rf: a --private-dir run of # .github/scripts/parallel-make-check.py replaces the certs symlink with a -# private directory copy, which rm -f would not remove. +# private directory copy, which rm -f would not remove. The recipe body +# runs under set -e so a failed symlink aborts the build instead of being +# stamped complete. ############################################################################## BUILT_SOURCES += wolfssl-test-data.stamp wolfssl-test-data.stamp: - $(AM_V_at)if test "$(abs_top_srcdir)" != "$(abs_top_builddir)"; then \ + $(AM_V_at)set -e; \ + if test "$(abs_top_srcdir)" != "$(abs_top_builddir)"; then \ $(MKDIR_P) tests scripts examples; \ for f in certs input quit; do \ rm -rf "$$f"; \ From 12597308dea190cce98d9c404ddf8b3985c7fdc2 Mon Sep 17 00:00:00 2001 From: Juliusz Sosinowicz Date: Thu, 11 Jun 2026 23:06:59 +0000 Subject: [PATCH 5/9] smoke-test, os-check: pin CCACHE_DIR to the cached path The two jobs that manage their ccache cache manually rely on ccache's XDG default (~/.cache/ccache) matching the actions/cache path. That holds today, but nothing enforces it: a later change that sets CCACHE_DIR (e.g. adopting the ccache-setup composite, which uses ~/.ccache) would silently decouple the build's cache from the saved/restored directory. Pin CCACHE_DIR explicitly to the cached path so the pairing is visible and cannot drift. --- .github/workflows/os-check.yml | 6 ++++++ .github/workflows/smoke-test.yml | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/.github/workflows/os-check.yml b/.github/workflows/os-check.yml index 10aa21d26b..5efd8954d0 100644 --- a/.github/workflows/os-check.yml +++ b/.github/workflows/os-check.yml @@ -75,6 +75,12 @@ jobs: - name: Allow unprivileged user namespaces (for bwrap) run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true + # ccache's default cache dir (XDG ~/.cache/ccache) is what the + # actions/cache step below saves; pin it explicitly so the two + # cannot drift apart (e.g. if a later change sets CCACHE_DIR). + - name: Pin ccache directory + run: echo "CCACHE_DIR=$HOME/.cache/ccache" >> "$GITHUB_ENV" + - name: Restore ccache uses: actions/cache@v4 with: diff --git a/.github/workflows/smoke-test.yml b/.github/workflows/smoke-test.yml index 3ab7bf6ab5..ec3c764f99 100644 --- a/.github/workflows/smoke-test.yml +++ b/.github/workflows/smoke-test.yml @@ -98,6 +98,13 @@ jobs: if: steps.merge_check.outputs.skip != 'true' run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true + # ccache's default cache dir (XDG ~/.cache/ccache) is what the + # actions/cache step below saves; pin it explicitly so the two + # cannot drift apart (e.g. if a later change sets CCACHE_DIR). + - name: Pin ccache directory + if: steps.merge_check.outputs.skip != 'true' + run: echo "CCACHE_DIR=$HOME/.cache/ccache" >> "$GITHUB_ENV" + - name: Restore ccache if: steps.merge_check.outputs.skip != 'true' uses: actions/cache@v4 From 85d3bc23807fe7752030fc5bc3fd21c75e7bb5c9 Mon Sep 17 00:00:00 2001 From: Juliusz Sosinowicz Date: Thu, 11 Jun 2026 23:42:17 +0000 Subject: [PATCH 6/9] parallel-make-check: drop the --jobs option wolfSSL's configure enables make's jobserver by default (AX_AM_JOBSERVER([yes]) -> AM_MAKEFLAGS += -j in aminclude.am), and automake passes that explicit -j to every recursive sub-make, where it overrides the invoking make's job limit. The script's -j therefore only ever scheduled the outermost recursion hop: --jobs was inert. Measured on a 4-CPU host with 10 build-only configs oversaturating the worker pool, the jobserver default is also the better policy: capping sub-makes via --disable-jobserver and -j2 dropped CPU utilization from 96% to 89% and lengthened the wall time, because configs' serial phases (configure, link) stopped being backfilled by other configs' compile jobs. So make is now invoked with no -j at all - parallelism within a config comes from the configure-default jobserver - and the misleading knob is gone, including the macOS job's --jobs 3. --- .github/scripts/parallel-make-check.py | 13 +++++++++---- .github/workflows/os-check.yml | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/scripts/parallel-make-check.py b/.github/scripts/parallel-make-check.py index 9395470e99..0b8061e871 100755 --- a/.github/scripts/parallel-make-check.py +++ b/.github/scripts/parallel-make-check.py @@ -238,7 +238,14 @@ def run_config(cfg, opts): configure.append(f"CC={cfg.cc}") flags = [f"CFLAGS={cfg.cflags}"] if cfg.cflags else [] flags += [f"LDFLAGS={cfg.ldflags}"] if cfg.ldflags else [] - make = ["make", f"-j{opts.jobs}"] + flags + # No -j here: wolfSSL's configure enables make's jobserver by default + # (AX_AM_JOBSERVER adds AM_MAKEFLAGS += -j), and that explicit + # -j on every automake sub-make overrides whatever the top-level make + # was given, so a -j here would only schedule the outermost recursion + # hop. Measured across this pool, the jobserver default also utilizes + # the CPUs better than a capped -j (configs' serial phases - configure, + # link - get backfilled by other configs' compile jobs). + make = ["make"] + flags steps = [] if cfg.user_settings: # Staged before configure; --enable-usersettings builds pick it up @@ -349,7 +356,7 @@ def summarize(results, wall_min, cpu_min, nthreads): # thread-minutes available (a long config left for last idles the other # workers and drags it down); CPU utilization is the CPU time the build # and test children actually consumed out of the CPU-minutes available - # (too-shallow make -j and serial test phases show up here). + # (serial configure/link/test phases show up here). busy_min = sum(minutes for _, _, minutes in results) ncpu = nproc() lines += [ @@ -380,8 +387,6 @@ def main(): p.add_argument("configs", nargs="*", metavar="NAME", help="configs to run (default: all)") p.add_argument("--list", action="store_true", help="list configs") - p.add_argument("--jobs", type=int, default=2, - help="make -j per config (default: 2)") p.add_argument("--threads", type=int, default=nproc(), help="worker threads; each takes the next pending config " "when it is free (default: nproc)") diff --git a/.github/workflows/os-check.yml b/.github/workflows/os-check.yml index 5efd8954d0..0e4b9a01b8 100644 --- a/.github/workflows/os-check.yml +++ b/.github/workflows/os-check.yml @@ -478,7 +478,7 @@ jobs: ] EOF .github/scripts/parallel-make-check.py \ - --threads 1 --jobs 3 --cc= \ + --threads 1 --cc= \ --cflags='-pedantic -Wdeclaration-after-statement -Wnull-dereference -Wno-overlength-strings -DTEST_LIBWOLFSSL_SOURCES_INCLUSION_SEQUENCE' \ --private-dir=certs "$RUNNER_TEMP/os-check-macos-configs.json" From 1b3a1ef6a6083c631cafde6762b2cb07d2edca86 Mon Sep 17 00:00:00 2001 From: Juliusz Sosinowicz Date: Fri, 12 Jun 2026 02:15:49 +0000 Subject: [PATCH 7/9] Fix CI: register JSON-config CPPFLAGS macros in .wolfssl_known_macro_extras The workflow matrix rework moved configure args into JSON lists, where CPPFLAGS defines appear as "CPPFLAGS=-DFOO". The known-macro scan in check-source-text only recognizes -D tokens preceded by whitespace or a quote, so macros whose only -D site now sits directly after the '=' are no longer registered as known. Add those macros to .wolfssl_known_macro_extras, and drop two entries that became redundant because the new workflow configs define them at a position the scanner does recognize. --- .wolfssl_known_macro_extras | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.wolfssl_known_macro_extras b/.wolfssl_known_macro_extras index 0369371fb2..101ebf2fa8 100644 --- a/.wolfssl_known_macro_extras +++ b/.wolfssl_known_macro_extras @@ -461,6 +461,8 @@ NO_TKERNEL_MEM_POOL NO_TLSX_PSKKEM_PLAIN_ANNOUNCE NO_VERIFY_OID NO_WC_DHGENERATEPUBLIC +NO_WC_SHE_GETUID +NO_WC_SHE_IMPORT_M123 NO_WC_SHE_LOADKEY NO_WC_SSIZE_TYPE NO_WOLFSSL_ALLOC_ALIGN @@ -631,6 +633,7 @@ USS_API WC_AESXTS_STREAM_NO_REQUEST_ACCOUNTING WC_AES_BS_WORD_SIZE WC_AES_GCM_DEC_AUTH_EARLY +WC_ALLOW_ECC_ZERO_HASH WC_ASN_HASH_SHA256 WC_ASN_RUNTIME_DATE_CHECK_CONTROL WC_ASYNC_ENABLE_ECC_KEYGEN @@ -678,6 +681,7 @@ WC_RNG_BLOCKING WC_RSA_NONBLOCK_TIME WC_RSA_NO_FERMAT_CHECK WC_RWLOCK_OPS_INLINE +WC_SHE_SW_DEFAULT WC_SKIP_INCLUDED_C_FILES WC_SLHDSA_KERNEL_ASM WC_SLHDSA_NO_ASM @@ -737,6 +741,7 @@ WOLFSSL_CAAM_BLACK_KEY_SM WOLFSSL_CAAM_NO_BLACK_KEY WOLFSSL_CALLBACKS WOLFSSL_CHECK_DESKEY +WOLFSSL_CHECK_MEM_ZERO WOLFSSL_CHIBIOS WOLFSSL_CLANG_TIDY WOLFSSL_CLIENT_EXAMPLE @@ -745,7 +750,9 @@ WOLFSSL_CRL_ALLOW_MISSING_CDP WOLFSSL_DISABLE_EARLY_SANITY_CHECKS WOLFSSL_DRBG_SHA256 WOLFSSL_DTLS13_ECHO_LEGACY_SESSION_ID +WOLFSSL_DTLS13_NO_HRR_ON_RESUME WOLFSSL_DTLS_DISALLOW_FUTURE +WOLFSSL_DTLS_NO_HVR_ON_RESUME WOLFSSL_DTLS_RECORDS_CAN_SPAN_DATAGRAMS WOLFSSL_DTLS_RESEND_ONLY_TIMEOUT WOLFSSL_DUMP_MEMIO_STREAM @@ -764,6 +771,7 @@ WOLFSSL_ESPWROOM32 WOLFSSL_EVP_PRINT WOLFSSL_EXPORT_INT WOLFSSL_EXPORT_SPC_SZ +WOLFSSL_EXTRA WOLFSSL_FORCE_OCSP_NONCE_CHECK WOLFSSL_FRDM_K64 WOLFSSL_FRDM_K64_JENKINS @@ -808,14 +816,17 @@ WOLFSSL_MANUALLY_SELECT_DEVICE_CONFIG WOLFSSL_MDK5 WOLFSSL_MEM_FAIL_COUNT WOLFSSL_MICROCHIP_AESGCM +WOLFSSL_MLKEM_DYNAMIC_KEYS WOLFSSL_MLKEM_INVNTT_UNROLL WOLFSSL_MLKEM_NO_MALLOC WOLFSSL_MLKEM_NTT_UNROLL +WOLFSSL_ML_KEM_USE_OLD_IDS WOLFSSL_MONT_RED_CT WOLFSSL_MP_COND_COPY WOLFSSL_MP_INVMOD_CONSTANT_TIME WOLFSSL_MULTICIRCULATE_ALTNAMELIST WOLFSSL_NEW_PRIME_CHECK +WOLFSSL_NONBLOCK_OCSP WOLFSSL_NOSHA3_384 WOLFSSL_NOT_WINDOWS_API WOLFSSL_NO_BIO_ADDR_IN @@ -970,8 +981,6 @@ WOLFSSL_XIL_MSG_NO_SLEEP WOLFSSL_ZEPHYR WOLF_ALLOW_BUILTIN WOLF_CRYPTO_CB_CMD -WOLF_CRYPTO_CB_NO_SHA512_FALLBACK -WOLF_CRYPTO_CB_ONLY_SHA512 WOLF_CRYPTO_DEV WOLF_NO_TRAILING_ENUM_COMMAS WindowsCE From 6d1d750ad3187005dfe0c042341f7d97dee197de Mon Sep 17 00:00:00 2001 From: Juliusz Sosinowicz Date: Fri, 12 Jun 2026 13:39:28 +0200 Subject: [PATCH 8/9] parallel-make-check: reserved names, type hints, readability - Reject the config names "aux" and "test": build-aux/ is autotools' aux-script dir and build-test/ a legacy build dir, neither the script's to wipe and rebuild over. - Add type hints throughout. - Reword the shard-partition comment (the LPT bound was unparseable) and replace the zip-over-pool.map result pairing with a run_one() helper so the pool returns complete result rows. --- .github/scripts/parallel-make-check.py | 58 ++++++++++++++++---------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/.github/scripts/parallel-make-check.py b/.github/scripts/parallel-make-check.py index 0b8061e871..3fd103fb76 100755 --- a/.github/scripts/parallel-make-check.py +++ b/.github/scripts/parallel-make-check.py @@ -10,6 +10,7 @@ # "name" (unknown keys are an error, so typos do not pass silently): # # name unique identifier; the config builds in build-/ +# ("aux" and "test" are reserved: build-aux/, build-test/) # configure list of extra ./configure arguments # cc compiler passed to configure as CC=, overriding --cc # ("" leaves CC entirely to configure / the environment) @@ -54,6 +55,8 @@ # fails fast; pass --no-fail-fast to run everything and report every # failure. +from __future__ import annotations + import argparse import json import os @@ -63,9 +66,11 @@ import subprocess import sys import threading import time +from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from pathlib import Path +from typing import NoReturn # cflags/ldflags are applied at make time only (never to ./configure) so # autoconf feature detection is not poisoned by benign warnings in @@ -74,15 +79,15 @@ from pathlib import Path @dataclass class Config: name: str - configure: list = field(default_factory=list) + configure: list[str] = field(default_factory=list) cc: str = "" cflags: str = "" ldflags: str = "" minutes: float = 1.0 user_settings: str = "" check: bool = True - prepare: list = field(default_factory=list) - run: list = field(default_factory=list) + prepare: list[list[str]] = field(default_factory=list) + run: list[list[str]] = field(default_factory=list) SRCDIR = Path(__file__).resolve().parents[2] ON_GITHUB = os.environ.get("GITHUB_ACTIONS") == "true" @@ -93,11 +98,11 @@ print_lock = threading.Lock() # workers' in-flight process groups. stop_event = threading.Event() fail_lock = threading.Lock() -live_procs = set() +live_procs: set[subprocess.Popen] = set() procs_lock = threading.Lock() -def kill_group(p, sig): +def kill_group(p: subprocess.Popen, sig: signal.Signals) -> None: # Every subprocess starts its own session, so signalling the process # group takes down the whole make/test tree under it. try: @@ -109,7 +114,7 @@ def kill_group(p, sig): pass -def abort_others(): +def abort_others() -> None: with procs_lock: procs = list(live_procs) for p in procs: @@ -126,7 +131,7 @@ def abort_others(): time.sleep(0.2) -def nproc(): +def nproc() -> int: # Like nproc(1): CPUs usable by this process, falling back to all online. try: return len(os.sched_getaffinity(0)) @@ -134,7 +139,8 @@ def nproc(): return os.cpu_count() or 1 -def load_configs(opts, error): +def load_configs(opts: argparse.Namespace, + error: Callable[[str], NoReturn]) -> list[Config]: try: if opts.json == "-": entries = json.load(sys.stdin) @@ -158,6 +164,12 @@ def load_configs(opts, error): if not isinstance(name, str) or not name or "/" in name: error(f"{opts.json}: every config needs a \"name\" usable as a " f"directory suffix: {entry!r}") + # build- dirs that are not ours to wipe: build-aux/ is + # autotools' aux-script dir (autogen.sh), build-test/ a legacy + # build dir (.gitignore). + if name in ("aux", "test"): + error(f"{opts.json}: reserved config name {name!r}: build-{name}/ " + f"belongs to other tooling") if any(cfg.name == name for cfg in configs): error(f"{opts.json}: duplicate config name {name!r}") configure = entry.get("configure", []) @@ -202,7 +214,7 @@ def load_configs(opts, error): return configs -def privatize_dirs(bdir, dirs): +def privatize_dirs(bdir: Path, dirs: list[str]) -> None: # Replace build-tree symlinks into the source tree with private # per-build-dir copies: tests that write into these directories would # otherwise write through the symlink into the shared source tree and @@ -215,7 +227,7 @@ def privatize_dirs(bdir, dirs): shutil.copytree(SRCDIR / name, d, symlinks=True) -def dump(title, path): +def dump(title: str, path: Path) -> None: print(f"::group::{title}" if ON_GITHUB else f"==== {title} ====") try: sys.stdout.write(path.read_text(errors="replace")) @@ -226,7 +238,8 @@ def dump(title, path): sys.stdout.flush() -def run_config(cfg, opts): +def run_config(cfg: Config, opts: argparse.Namespace) -> tuple[str | None, + float]: if opts.fail_fast and stop_event.is_set(): return "aborted", 0.0 bdir = SRCDIR / f"build-{cfg.name}" @@ -246,7 +259,7 @@ def run_config(cfg, opts): # the CPUs better than a capped -j (configs' serial phases - configure, # link - get backfilled by other configs' compile jobs). make = ["make"] + flags - steps = [] + steps: list[tuple[str, list[str] | Callable[[], object]]] = [] if cfg.user_settings: # Staged before configure; --enable-usersettings builds pick it up # from the build dir via the default include path. @@ -264,11 +277,11 @@ def run_config(cfg, opts): ("make check", ["make"] + flags + ["check"]), ] steps += [(" ".join(cmd), cmd) for cmd in cfg.run] - failed = None + failed: str | None = None start = time.monotonic() log = bdir / "make-check.log" - def record_failure(step): + def record_failure(step: str) -> str: # Classify a failed step, doing the fail-fast bookkeeping: the # first failure wins and aborts everyone else; any failure after # the abort began is reported as aborted instead. @@ -341,7 +354,8 @@ def run_config(cfg, opts): return failed, minutes -def summarize(results, wall_min, cpu_min, nthreads): +def summarize(results: list[tuple[Config, str | None, float]], + wall_min: float, cpu_min: float, nthreads: int) -> None: lines = ["| Config | Result | Minutes |", "|---|---|---|"] for cfg, failed, minutes in results: if failed == "aborted": @@ -376,7 +390,7 @@ def summarize(results, wall_min, cpu_min, nthreads): print(f"### make check\n\n{table}", file=f) -def main(): +def main() -> int: p = argparse.ArgumentParser( description="Build and make check every configuration from a JSON " "file in its own out-of-tree build directory, in " @@ -436,8 +450,8 @@ def main(): p.error(f"--shard: expected K/N with 1 <= K <= N, " f"got {opts.shard!r}") # Greedy multiway partition: longest first into the least-loaded - # shard. Deterministic, and with honest "minutes" within ~the - # longest config of optimal. + # shard. Deterministic; if the "minutes" are accurate, the worst + # shard ends up within about one config's minutes of optimal. shards, loads = [[] for _ in range(n)], [0.0] * n for cfg in selected: i = loads.index(min(loads)) @@ -460,10 +474,12 @@ def main(): nthreads = max(1, min(opts.threads, len(selected))) wall_start = time.monotonic() cpu_start = os.times() + def run_one(cfg: Config) -> tuple[Config, str | None, float]: + failed, minutes = run_config(cfg, opts) + return cfg, failed, minutes + with ThreadPoolExecutor(max_workers=nthreads) as pool: - results = [(cfg, failed, minutes) for cfg, (failed, minutes) - in zip(selected, pool.map( - lambda cfg: run_config(cfg, opts), selected))] + results = list(pool.map(run_one, selected)) wall_min = (time.monotonic() - wall_start) / 60 cpu_end = os.times() # os.times() child counters cover the waited-for configure/make From 742a0267f20e9deecfe4c52472fcb86431ce20a1 Mon Sep 17 00:00:00 2001 From: Juliusz Sosinowicz Date: Fri, 12 Jun 2026 14:12:56 +0200 Subject: [PATCH 9/9] trackmemory: raise timeout to 40 min for wolfEntropy runtime variance --- .github/workflows/trackmemory.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/trackmemory.yml b/.github/workflows/trackmemory.yml index ea04c96d82..6d040a78ee 100644 --- a/.github/workflows/trackmemory.yml +++ b/.github/workflows/trackmemory.yml @@ -25,8 +25,10 @@ jobs: name: make check if: ${{ (github.repository_owner == 'wolfssl') && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} runs-on: ubuntu-24.04 - # Generous for a cold ccache; warm reruns finish in a fraction. - timeout-minutes: 20 + # Typical runs finish in 13-16 min, but the wolfEntropy configs gather + # entropy from CPU timing jitter, whose runtime varies severalfold with + # runner contention (a 20-min limit was hit with one config left). + timeout-minutes: 40 steps: - uses: actions/checkout@v4 name: Checkout wolfSSL