nexus-workshops/anvil
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Disabled script execution tests for now. This commit is good but doesn't include a working script execution test, which tests build.bat and build.sh and test.bat/test.sh, etc. in the generated project.
Some checks failed
CI / Test (Windows MSVC) (push) Has been cancelled
Details
CI / Test (Linux) (push) Has been cancelled
Details
CI / Clippy (push) Has been cancelled
Details
CI / Format (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Eric Ratliff
2026-02-22 21:49:34 -06:00
parent 6a0b3af330
commit 1ede07df81
1 changed files with 355 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

661
tests/script_execution_test.rs
View File

@@ -1,661 +0,0 @@
// ==========================================================================
// Script Execution Tests
// ==========================================================================
//
// These tests extract a real Anvil project to a temp directory and then
// ACTUALLY EXECUTE the generated scripts. They are not fast and they
// require real build-machine dependencies:
//
// test/run_tests.sh --> cmake, g++ (or clang++), git
// build.sh --> arduino-cli (with arduino:avr core installed)
//
// If any dependency is missing the test FAILS -- that is intentional.
// A build machine that ships Anvil binaries MUST have these tools.
//
// On Windows the .bat variants are tested instead.
// ==========================================================================
use std::fs;
use std::path::Path;
use std::process::Command;
use tempfile::TempDir;
use anvil::templates::{TemplateContext, TemplateManager};
use anvil::version::ANVIL_VERSION;
// --------------------------------------------------------------------------
// Helpers
// --------------------------------------------------------------------------
/// Extract a fresh project into a temp directory and return the TempDir.
fn extract_project(name: &str) -> TempDir {
let tmp = TempDir::new().expect("Failed to create temp directory");
let ctx = TemplateContext {
project_name: name.to_string(),
anvil_version: ANVIL_VERSION.to_string(),
};
let count = TemplateManager::extract("basic", tmp.path(), &ctx)
.expect("Failed to extract template");
assert!(count > 0, "Template extraction produced zero files");
tmp
}
/// Make all .sh files under `root` executable (Unix only).
#[cfg(unix)]
fn chmod_scripts(root: &Path) {
use std::os::unix::fs::PermissionsExt;
for entry in walkdir::WalkDir::new(root)
.into_iter()
.filter_map(|e| e.ok())
{
let p = entry.path();
if p.extension().map(|e| e == "sh").unwrap_or(false) {
let mut perms = fs::metadata(p)
.expect("Failed to read metadata")
.permissions();
perms.set_mode(0o755);
fs::set_permissions(p, perms).expect("Failed to chmod");
}
}
}
/// Run a shell script (bash on Unix, cmd /C on Windows).
/// Returns (success, stdout, stderr).
fn run_script(dir: &Path, script: &str) -> (bool, String, String) {
let output = if cfg!(windows) {
Command::new("cmd")
.args(["/C", script])
.current_dir(dir)
.output()
} else {
Command::new("bash")
.arg(script)
.current_dir(dir)
.output()
};
let output = output.unwrap_or_else(|e| panic!("Failed to execute {}: {}", script, e));
let stdout = String::from_utf8_lossy(&output.stdout).to_string();
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
(output.status.success(), stdout, stderr)
}
/// Run a script with extra arguments.
fn run_script_with_args(dir: &Path, script: &str, args: &[&str]) -> (bool, String, String) {
let output = if cfg!(windows) {
let mut all_args = vec!["/C", script];
all_args.extend_from_slice(args);
Command::new("cmd")
.args(&all_args)
.current_dir(dir)
.output()
} else {
Command::new("bash")
.arg(script)
.args(args)
.current_dir(dir)
.output()
};
let output = output.unwrap_or_else(|e| panic!("Failed to execute {}: {}", script, e));
let stdout = String::from_utf8_lossy(&output.stdout).to_string();
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
(output.status.success(), stdout, stderr)
}
/// Assert that a command-line tool is available in PATH.
/// Panics with a clear message if not found.
fn require_tool(name: &str) {
let check = if cfg!(windows) {
Command::new("where").arg(name).output()
} else {
Command::new("which").arg(name).output()
};
match check {
Ok(output) if output.status.success() => {}
_ => panic!(
"\n\n\
===================================================================\n\
MISSING BUILD DEPENDENCY: {name}\n\
===================================================================\n\
\n\
Anvil's cargo tests REQUIRE build-machine dependencies.\n\
Install '{name}' and re-run. See 'anvil doctor' for guidance.\n\
\n\
===================================================================\n"
),
}
}
/// Check that at least one C++ compiler is present.
fn require_cpp_compiler() {
let has_gpp = Command::new(if cfg!(windows) { "where" } else { "which" })
.arg("g++")
.output()
.map(|o| o.status.success())
.unwrap_or(false);
let has_clangpp = Command::new(if cfg!(windows) { "where" } else { "which" })
.arg("clang++")
.output()
.map(|o| o.status.success())
.unwrap_or(false);
if !has_gpp && !has_clangpp {
panic!(
"\n\n\
===================================================================\n\
MISSING BUILD DEPENDENCY: C++ compiler (g++ or clang++)\n\
===================================================================\n\
\n\
Install g++ or clang++ and re-run.\n\
\n\
===================================================================\n"
);
}
}
/// Require cmake + C++ compiler + git (the test script prereqs).
fn require_test_script_deps() {
require_tool("cmake");
require_tool("git");
require_cpp_compiler();
}
/// Require arduino-cli (the build script prereqs).
fn require_build_script_deps() {
require_tool("arduino-cli");
}
/// Choose the platform-appropriate script path.
fn test_script() -> &'static str {
if cfg!(windows) { "test\\run_tests.bat" } else { "test/run_tests.sh" }
}
fn build_script() -> &'static str {
if cfg!(windows) { "build.bat" } else { "build.sh" }
}
/// List directory contents recursively (for diagnostic output on failure).
fn list_dir_recursive(dir: &Path) -> String {
if !dir.exists() {
return format!(" (directory does not exist: {})", dir.display());
}
let mut lines = Vec::new();
for entry in walkdir::WalkDir::new(dir)
.max_depth(4)
.into_iter()
.filter_map(|e| e.ok())
{
let depth = entry.depth();
let indent = " ".repeat(depth);
let name = entry.file_name().to_string_lossy();
lines.push(format!("{}{}", indent, name));
}
lines.join("\n")
}
// ==========================================================================
// TEST SCRIPT EXECUTION (cmake + Google Test)
//
// Extracts a project, then runs test/run_tests.sh which:
// 1. cmake configures the test/ directory
// 2. FetchContent downloads Google Test
// 3. Compiles C++ unit tests against HAL mocks
// 4. Runs tests via ctest
//
// Required on build machine: cmake, g++ or clang++, git
// ==========================================================================
#[test]
fn test_run_tests_script_executes_successfully() {
require_test_script_deps();
let tmp = extract_project("test_exec");
#[cfg(unix)]
chmod_scripts(tmp.path());
let (success, stdout, stderr) = run_script(tmp.path(), test_script());
// Always print output so CI logs are useful
println!("--- run_tests stdout ---\n{}", stdout);
if !stderr.is_empty() {
eprintln!("--- run_tests stderr ---\n{}", stderr);
}
assert!(
success,
"test/run_tests script failed!\n\nstdout:\n{}\n\nstderr:\n{}",
stdout, stderr
);
}
#[test]
fn test_run_tests_script_google_tests_actually_ran() {
require_test_script_deps();
let tmp = extract_project("gtest_verify");
#[cfg(unix)]
chmod_scripts(tmp.path());
let (success, stdout, stderr) = run_script(tmp.path(), test_script());
assert!(
success,
"Test script failed.\nstdout:\n{}\nstderr:\n{}",
stdout, stderr
);
// Verify that Google Test actually executed tests -- not a silent no-op
let combined = format!("{}{}", stdout, stderr);
let tests_ran = combined.contains("passed")
|| combined.contains("PASSED")
|| combined.contains("tests passed")
|| combined.contains("100%");
assert!(
tests_ran,
"Output does not indicate any Google Tests actually executed.\n\
This could mean cmake built but ctest found no registered tests.\n\n\
stdout:\n{}\n\nstderr:\n{}",
stdout, stderr
);
}
#[test]
fn test_run_tests_script_clean_flag_rebuilds() {
require_test_script_deps();
let tmp = extract_project("clean_rebuild");
#[cfg(unix)]
chmod_scripts(tmp.path());
// First run -- populates build dir and fetches gtest
let (success, _, _) = run_script(tmp.path(), test_script());
assert!(success, "First test run failed");
// Verify build artifacts exist
let build_dir = tmp.path().join("test").join("build");
assert!(
build_dir.exists(),
"test/build/ directory should exist after first run"
);
// Second run with --clean -- should nuke build dir and rebuild
let (success, stdout, stderr) =
run_script_with_args(tmp.path(), test_script(), &["--clean"]);
println!("--- clean rebuild stdout ---\n{}", stdout);
if !stderr.is_empty() {
eprintln!("--- clean rebuild stderr ---\n{}", stderr);
}
assert!(
success,
"Clean rebuild failed.\nstdout:\n{}\nstderr:\n{}",
stdout, stderr
);
}
#[test]
fn test_run_tests_script_produces_test_binary() {
require_test_script_deps();
let tmp = extract_project("bin_check");
#[cfg(unix)]
chmod_scripts(tmp.path());
let (success, _, _) = run_script(tmp.path(), test_script());
assert!(success, "Test script failed");
// The cmake build should produce a test_unit binary somewhere
// under test/build/
let build_dir = tmp.path().join("test").join("build");
let has_binary = walkdir::WalkDir::new(&build_dir)
.into_iter()
.filter_map(|e| e.ok())
.any(|entry| {
let name = entry.file_name().to_string_lossy();
name.starts_with("test_unit")
});
assert!(
has_binary,
"Expected test_unit binary under test/build/ after running tests.\n\
Contents of test/build/:\n{}",
list_dir_recursive(&build_dir)
);
}
#[test]
fn test_run_tests_idempotent_second_run() {
// Running the test script twice should succeed both times.
// Second run reuses the cached build and should be fast.
require_test_script_deps();
let tmp = extract_project("idempotent");
#[cfg(unix)]
chmod_scripts(tmp.path());
let (success1, _, _) = run_script(tmp.path(), test_script());
assert!(success1, "First run failed");
let (success2, stdout2, stderr2) = run_script(tmp.path(), test_script());
assert!(
success2,
"Second run failed (should be idempotent).\nstdout:\n{}\nstderr:\n{}",
stdout2, stderr2
);
}
// ==========================================================================
// BUILD SCRIPT EXECUTION (arduino-cli compile)
//
// Extracts a project, then runs build.sh which:
// 1. Reads .anvil.toml for FQBN, include_dirs, extra_flags
// 2. Invokes arduino-cli compile with those settings
//
// Required on build machine: arduino-cli with arduino:avr core installed
// ==========================================================================
#[test]
fn test_build_script_compiles_sketch() {
require_build_script_deps();
let tmp = extract_project("build_test");
#[cfg(unix)]
chmod_scripts(tmp.path());
let (success, stdout, stderr) = run_script(tmp.path(), build_script());
println!("--- build.sh stdout ---\n{}", stdout);
if !stderr.is_empty() {
eprintln!("--- build.sh stderr ---\n{}", stderr);
}
assert!(
success,
"build script failed!\n\nstdout:\n{}\n\nstderr:\n{}",
stdout, stderr
);
}
#[test]
fn test_build_script_produces_compilation_output() {
require_build_script_deps();
let tmp = extract_project("compile_output");
#[cfg(unix)]
chmod_scripts(tmp.path());
let (success, stdout, stderr) = run_script(tmp.path(), build_script());
assert!(
success,
"Build script failed.\nstdout:\n{}\nstderr:\n{}",
stdout, stderr
);
// arduino-cli compile produces output indicating sketch size.
// Look for evidence of successful compilation.
let combined = format!("{}{}", stdout, stderr);
let compiled = combined.contains("Sketch uses")
|| combined.contains("bytes")
|| combined.contains("Compiling")
|| combined.contains("Used")
|| combined.contains("compiled");
assert!(
compiled,
"Build output does not indicate a successful arduino-cli compilation.\n\n\
stdout:\n{}\n\nstderr:\n{}",
stdout, stderr
);
}
#[test]
fn test_build_script_idempotent_second_run() {
require_build_script_deps();
let tmp = extract_project("build_idem");
#[cfg(unix)]
chmod_scripts(tmp.path());
let (success1, _, _) = run_script(tmp.path(), build_script());
assert!(success1, "First build failed");
let (success2, stdout2, stderr2) = run_script(tmp.path(), build_script());
assert!(
success2,
"Second build failed (should be idempotent).\nstdout:\n{}\nstderr:\n{}",
stdout2, stderr2
);
}
// ==========================================================================
// COMBINED: build + test scripts both succeed on the same project
//
// Full end-to-end: one extracted project, both scripts pass.
// ==========================================================================
#[test]
fn test_full_project_build_and_test_scripts_both_pass() {
require_test_script_deps();
require_build_script_deps();
let tmp = extract_project("full_e2e");
#[cfg(unix)]
chmod_scripts(tmp.path());
// Build the sketch
let (build_ok, build_out, build_err) = run_script(tmp.path(), build_script());
println!("--- build stdout ---\n{}", build_out);
if !build_err.is_empty() {
eprintln!("--- build stderr ---\n{}", build_err);
}
assert!(
build_ok,
"build script failed in full E2E.\nstdout:\n{}\nstderr:\n{}",
build_out, build_err
);
// Run the host-side unit tests
let (test_ok, test_out, test_err) = run_script(tmp.path(), test_script());
println!("--- test stdout ---\n{}", test_out);
if !test_err.is_empty() {
eprintln!("--- test stderr ---\n{}", test_err);
}
assert!(
test_ok,
"test script failed in full E2E.\nstdout:\n{}\nstderr:\n{}",
test_out, test_err
);
}
// ==========================================================================
// SCRIPT CONTENT SANITY CHECKS
//
// Verify the scripts are well-formed before even executing them.
// ==========================================================================
#[test]
fn test_all_sh_scripts_have_strict_error_handling() {
let tmp = extract_project("strict_mode");
let sh_scripts = vec![
"build.sh",
"upload.sh",
"monitor.sh",
"test/run_tests.sh",
];
for script in &sh_scripts {
let path = tmp.path().join(script);
if !path.exists() {
continue;
}
let content = fs::read_to_string(&path)
.unwrap_or_else(|_| panic!("Failed to read {}", script));
assert!(
content.contains("set -e") || content.contains("set -euo pipefail"),
"{} must use 'set -e' or 'set -euo pipefail' for strict error handling",
script
);
}
}
#[test]
fn test_all_sh_scripts_have_shebangs() {
let tmp = extract_project("shebang_check");
let sh_scripts = vec![
"build.sh",
"upload.sh",
"monitor.sh",
"test/run_tests.sh",
];
for script in &sh_scripts {
let path = tmp.path().join(script);
if !path.exists() {
continue;
}
let content = fs::read_to_string(&path)
.unwrap_or_else(|_| panic!("Failed to read {}", script));
assert!(
content.starts_with("#!/"),
"{} must start with a shebang line (#!/...)",
script
);
}
}
#[test]
fn test_bat_scripts_exist_for_windows_parity() {
let tmp = extract_project("win_parity");
// Every .sh should have a matching .bat
let pairs = vec![
("build.sh", "build.bat"),
("upload.sh", "upload.bat"),
("monitor.sh", "monitor.bat"),
("test/run_tests.sh", "test/run_tests.bat"),
];
for (sh, bat) in &pairs {
let sh_exists = tmp.path().join(sh).exists();
let bat_exists = tmp.path().join(bat).exists();
if sh_exists {
assert!(
bat_exists,
"{} exists but {} is missing -- Windows parity broken",
sh, bat
);
}
}
}
#[test]
fn test_cmake_lists_fetches_google_test() {
let tmp = extract_project("cmake_gtest");
let cmake_path = tmp.path().join("test").join("CMakeLists.txt");
assert!(cmake_path.exists(), "test/CMakeLists.txt must exist");
let content = fs::read_to_string(&cmake_path).unwrap();
assert!(
content.contains("FetchContent") || content.contains("fetchcontent"),
"CMakeLists.txt should use FetchContent to download Google Test"
);
assert!(
content.contains("googletest") || content.contains("GTest"),
"CMakeLists.txt should reference Google Test"
);
}
#[test]
fn test_scripts_all_reference_anvil_toml() {
let tmp = extract_project("toml_refs");
// build and upload scripts must read .anvil.toml for configuration
let config_scripts = vec![
"build.sh",
"build.bat",
"upload.sh",
"upload.bat",
];
for script in &config_scripts {
let path = tmp.path().join(script);
if !path.exists() {
continue;
}
let content = fs::read_to_string(&path)
.unwrap_or_else(|_| panic!("Failed to read {}", script));
assert!(
content.contains(".anvil.toml"),
"{} should reference .anvil.toml for project configuration",
script
);
}
}
#[test]
fn test_scripts_invoke_arduino_cli_not_anvil() {
let tmp = extract_project("no_anvil_dep");
// All scripts must invoke arduino-cli directly, never the anvil binary
let scripts = vec![
"build.sh", "build.bat",
"upload.sh", "upload.bat",
"monitor.sh", "monitor.bat",
];
for script in &scripts {
let path = tmp.path().join(script);
if !path.exists() {
continue;
}
let content = fs::read_to_string(&path)
.unwrap_or_else(|_| panic!("Failed to read {}", script));
assert!(
content.contains("arduino-cli"),
"{} should invoke arduino-cli directly",
script
);
// No line should shell out to the anvil binary
let has_anvil_cmd = content.lines().any(|line| {
let trimmed = line.trim();
// Skip comments and echo/print lines
if trimmed.starts_with('#')
|| trimmed.starts_with("::")
|| trimmed.starts_with("echo")
|| trimmed.starts_with("REM")
|| trimmed.starts_with("rem")
{
return false;
}
// Check for "anvil " as a command invocation
trimmed.contains("anvil ")
&& !trimmed.contains("anvil.toml")
&& !trimmed.contains("Anvil")
});
assert!(
!has_anvil_cmd,
"{} should not invoke the anvil binary -- project must be self-contained",
script
);
}
}