laantungir/ngit-grasp
1
0
Fork 0
mirror of https://relay.ngit.dev/npub15qydau2hjma6ngxkl2cyar74wzyjshvl65za5k5rl69264ar2exs5cyejr/ngit-grasp.git synced 2026-09-14 04:45:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge #cc893bf3: feat(grasp-audit): emit full audit JSON reports

Browse Source 
nostr:nevent1qgsx2lyl2e4zvfadwcvkd9fkrcwczj7mf858hy85mwqclwgut8wpg2spz3mhxue69uhhyetvv9ujumn8d96zuer9wcq3yamnwvaz7tm8d96xummnw3ezucm0d5q3kamnwvaz7tmwva5hgtnyv9hxxmmwwashjer9wchxxmmdqqsvezfm7w48p7z2sk05xs48pnjx7fddys0apezs9yvg7zrp9l7f9rsmathdx

PR-Author: DanConwayDev's Agent
nostr:npub1v47f74n2ycn66asev62nv8sas99akj0g0wg0fkup37u3ckwuzs4q7cwtp0

PR description:

Expose the merged stable audit result model through audit --json using one schema-versioned run envelope. Completed audits include tool/run metadata, pinned specification revisions, aggregate counts, and per-test outcomes. Parse, setup, and runtime failures emit the same envelope with an error outcome.

Human output and audit ordering remain intact. Exit status is 0 for pass, 1 for completed test failures, and 2 for invocation or runtime errors.
This commit is contained in:
DanConwayDev
2026-08-05 14:42:35 +01:00
parent 3d655ab605 c9b3bfaaf3
commit 1c586d269c
5 changed files with 620 additions and 243 deletions
Download Patch File Download Diff File Expand all files Collapse all files
grasp-audit/README.md
+10
View File
@@ -90,8 +90,18 @@ grasp-audit audit --relay ws://localhost:7334 --spec nip01-smoke
# Run with isolated fixtures (for testing/debugging)
grasp-audit audit --relay ws://localhost:7334 --mode isolated --spec push-auth
# Emit one stable JSON document for a scheduled full audit
grasp-audit audit --relay wss://relay.ngit.dev --spec all --json
```
`audit --json` writes one schema-versioned document to stdout and suppresses
the human progress/report output. Completed runs include tool and run metadata,
pinned GRASP revisions, aggregate counts, and per-test outcomes. Invalid
arguments and setup/runtime errors use the same envelope with an `error`
outcome. Exit status is `0` for a passing audit, `1` for completed test
failures, and `2` for invocation or runtime errors.
### Audit identity
Probe write checks and full audits can use an explicit identity. Key sources
grasp-audit/src/bin/grasp-audit.rs
+374 -237
View File
@@ -1,13 +1,13 @@
//! GRASP Audit CLI Tool
use clap::{Args, CommandFactory, Parser, Subcommand};
use clap::{Args, CommandFactory, Parser, Subcommand, ValueEnum};
use grasp_audit::*;
use std::ffi::OsString;
use std::path::PathBuf;
use std::time::Duration;
use std::time::{Duration, Instant};
#[derive(Parser)]
#[command(name = "grasp-audit")]
#[command(name = "grasp-audit", version)]
#[command(about = "GRASP audit and compliance testing tool", long_about = None)]
struct Cli {
#[command(subcommand)]
@@ -26,6 +26,30 @@ struct KeyArgs {
nsec_file: Option<PathBuf>,
}
#[derive(Debug, Clone, Copy, ValueEnum)]
enum AuditSelection {
Nip01Smoke,
Nip11,
EventAcceptance,
Cors,
GitClone,
GitFilter,
PushAuth,
RepoCreation,
Purgatory,
Grasp06,
All,
}
impl std::fmt::Display for AuditSelection {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let value = self
.to_possible_value()
.expect("audit selection variants have possible values");
formatter.write_str(value.get_name())
}
}
#[derive(Subcommand)]
enum Commands {
/// Run a probe/smoke test against a server
@@ -67,20 +91,21 @@ enum Commands {
relay: String,
/// Fixture mode: shared (default) or isolated
///
/// - shared: Fixtures are cached and reused across tests (efficient for sequential test runs)
/// - isolated: Each test creates fresh fixtures (for parallel tests like cargo test)
#[arg(short, long, default_value = "shared")]
mode: String,
/// Spec to test (nip01-smoke, nip11, event-acceptance, cors, git-clone, git-filter, push-auth, repo-creation, purgatory, grasp06, all)
#[arg(short, long, default_value = "all")]
spec: String,
/// Audit specification or suite to run.
#[arg(short, long, value_enum, default_value_t = AuditSelection::All)]
spec: AuditSelection,
/// Git data directory (required for cors, git-clone, push-auth, repo-creation specs)
#[arg(short, long)]
git_data_dir: Option<PathBuf>,
/// Output one stable machine-readable JSON document.
#[arg(long, default_value_t = false)]
json: bool,
#[command(flatten)]
keys: KeyArgs,
},
@@ -88,13 +113,11 @@ enum Commands {
#[tokio::main]
async fn main() -> Result<()> {
// Probe output is self-contained — library chatter (nostr_relay_pool etc.)
// adds no value and clutters both human and JSON output. Skip the tracing
// subscriber entirely for the probe subcommand; initialise it normally for
// audit subcommands where verbose output is expected.
let is_probe = std::env::args().nth(1).as_deref() == Some("probe");
let args: Vec<OsString> = std::env::args_os().collect();
let is_probe = args.get(1).is_some_and(|arg| arg == "probe");
if !is_probe {
tracing_subscriber::fmt()
.with_writer(std::io::stderr)
.with_env_filter(
tracing_subscriber::EnvFilter::from_default_env()
.add_directive(tracing::Level::INFO.into()),
@@ -102,7 +125,33 @@ async fn main() -> Result<()> {
.init();
}
let cli = Cli::parse();
let audit_json = audit_json_requested(&args);
let parse_started_at = unix_timestamp();
let parse_started = Instant::now();
let cli = match Cli::try_parse_from(&args) {
Ok(cli) => cli,
Err(error)
if matches!(
error.kind(),
clap::error::ErrorKind::DisplayHelp | clap::error::ErrorKind::DisplayVersion
) =>
{
error.print()?;
return Ok(());
}
Err(error) if audit_json => {
let report = AuditReport::error(
requested_relay(&args),
"",
parse_started_at,
parse_started.elapsed(),
format!("invalid command line ({:?})", error.kind()),
);
println!("{}", serde_json::to_string(&report)?);
std::process::exit(2);
}
Err(error) => error.exit(),
};
match cli.command {
Commands::Probe {
@@ -115,36 +164,26 @@ async fn main() -> Result<()> {
harden_network,
} => {
let relay = match relay {
Some(r) => r,
Some(relay) => relay,
None => {
// Print probe-specific help and exit cleanly
let mut cmd = Cli::command();
let _ = cmd.find_subcommand_mut("probe").unwrap().print_help();
let mut command = Cli::command();
let _ = command.find_subcommand_mut("probe").unwrap().print_help();
println!();
return Ok(());
}
};
let keys = load_keys(&keys)?;
// read_only is the default; --create-repo opts into the write path
let read_only = !create_repo;
// Overall probe timeout: min(20s, watch_interval) to prevent
// overlapping runs under --watch or cron scheduling.
let overall_secs = match watch {
Some(interval) => interval.min(20),
None => 20,
};
let overall_secs = watch.map_or(20, |interval| interval.min(20));
let options = ProbeOptions { harden_network };
if let Some(interval) = watch {
let mut run = 1u64;
let mut run = 1_u64;
loop {
if !json {
println!("\n[Run {}]", run);
println!("\n[Run {run}]");
}
let report = grasp_audit::probe::run_probe_with_options(
let report = run_probe_with_options(
&relay,
keys.clone(),
read_only,
@@ -162,15 +201,9 @@ async fn main() -> Result<()> {
tokio::time::sleep(Duration::from_secs(interval)).await;
}
} else {
let report = grasp_audit::probe::run_probe_with_options(
&relay,
keys,
read_only,
timeout,
overall_secs,
options,
)
.await;
let report =
run_probe_with_options(&relay, keys, read_only, timeout, overall_secs, options)
.await;
if json {
report.print_json();
} else {
@@ -181,213 +214,261 @@ async fn main() -> Result<()> {
}
}
}
Commands::Audit {
relay,
mode,
spec,
git_data_dir,
json,
keys,
} => {
let keys = load_keys(&keys)?;
let mut config = match mode.as_str() {
"shared" => AuditConfig::shared(),
"isolated" => AuditConfig::isolated(),
// Backwards compatibility aliases
"ci" => AuditConfig::isolated(),
"production" => AuditConfig::shared(),
_ => {
return Err(anyhow!(
"Invalid mode: {}. Use 'shared' or 'isolated'",
mode
))
}
};
// Audit needs to create events to test the relay, so disable read-only mode
config.read_only = false;
// Derive relay_domain from relay URL (e.g., "ws://localhost:8081" -> "localhost:8081")
let relay_domain = relay
.replace("ws://", "")
.replace("wss://", "")
.trim_end_matches('/')
.to_string();
println!("🔍 GRASP Audit Tool");
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!("Relay: {}", relay);
println!("Mode: {}", mode);
println!("Spec: {}", spec);
println!("Run ID: {}", config.run_id);
if let Some(ref dir) = git_data_dir {
println!("Git Dir: {}", dir.display());
}
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!();
println!("Connecting to relay...");
let client = match keys {
Some(keys) => AuditClient::new_with_keys(&relay, config, keys).await,
None => AuditClient::new(&relay, config).await,
}
.map_err(|e| anyhow!("Failed to connect to relay: {}", e))?;
if !client.is_connected().await {
return Err(anyhow!("Could not establish connection to relay"));
}
println!("✓ Connected\n");
let results = match spec.as_str() {
"nip01-smoke" => {
println!("Running NIP-01 smoke tests...\n");
specs::Nip01SmokeTests::run_all(&client).await
}
"nip11" => {
println!("Running NIP-11 document tests...\n");
specs::Nip11DocumentTests::run_all(&client).await
}
"event-acceptance" => {
println!("Running event acceptance policy tests...\n");
specs::EventAcceptancePolicyTests::run_all(&client).await
}
"cors" => {
println!("Running CORS tests...\n");
specs::CorsTests::run_all(&client, &relay_domain).await
}
"git-clone" => {
println!("Running Git clone tests...\n");
specs::GitCloneTests::run_all(&client, &relay_domain).await
}
"git-filter" => {
println!("Running Git filter capability tests...\n");
specs::GitFilterTests::run_all(&client, &relay_domain).await
}
"push-auth" => {
println!("Running push authorization tests...\n");
specs::PushAuthorizationTests::run_all(&client, &relay_domain).await
}
"repo-creation" => {
println!("Running repository creation tests...\n");
specs::RepositoryCreationTests::run_all(&client, &relay_domain).await
}
"purgatory" => {
println!("Running purgatory tests...\n");
specs::PurgatoryTests::run_all(&client).await
}
"grasp06" => {
println!("Running GRASP-06 tests...\n");
// GRASP-06 has its own report renderer (sections differ
// from GRASP-01). Print it directly and short-circuit the
// shared print_report path below.
let grasp06_results = specs::Grasp06Tests::run_all(&client).await;
specs::Grasp06Tests::print_report(&grasp06_results);
if !grasp06_results.all_passed() {
println!("❌ Some tests failed");
std::process::exit(1);
} else {
println!("✅ All tests passed!");
}
return Ok(());
}
"all" => {
println!("Running all tests...\n");
let mut all_results = AuditResult::new("All GRASP-01 Tests");
// NIP-01 smoke tests (stateless - no shared fixture dependencies)
println!(" → NIP-01 smoke tests...");
let nip01_results = specs::Nip01SmokeTests::run_all(&client).await;
all_results.merge(nip01_results);
// NIP-11 document tests (stateless)
println!(" → NIP-11 document tests...");
let nip11_results = specs::Nip11DocumentTests::run_all(&client).await;
all_results.merge(nip11_results);
// CORS tests (stateless HTTP checks)
println!(" → CORS tests...");
let cors_results = specs::CorsTests::run_all(&client, &relay_domain).await;
all_results.merge(cors_results);
// Repository creation tests (uses ValidRepoSent only - no state events)
println!(" → Repository creation tests...");
let repo_results = specs::RepositoryCreationTests::run_all(&client, &relay_domain).await;
all_results.merge(repo_results);
// Git clone tests (uses ValidRepoSent only - no state events)
println!(" → Git clone tests...");
let clone_results = specs::GitCloneTests::run_all(&client, &relay_domain).await;
all_results.merge(clone_results);
// Git filter capability tests (uses ValidRepoSent only - no state events)
println!(" → Git filter capability tests...");
let filter_results = specs::GitFilterTests::run_all(&client, &relay_domain).await;
all_results.merge(filter_results);
// Event acceptance policy tests (uses ValidRepoServed - no extra state events)
println!(" → Event acceptance policy tests...");
let event_results = specs::EventAcceptancePolicyTests::run_all(&client).await;
all_results.merge(event_results);
// Purgatory tests MUST run before push-auth.
// Push-auth sends new replaceable state events (kind 30618) for the same
// repo_id as OwnerStateDataPushed (e.g. test_head_set_after_git_push_with_required_oids
// sends a develop1 state event that displaces the original). If purgatory ran
// after push-auth, is_event_on_relay(original_id) would return false because
// the original state event has been replaced on the relay.
println!(" → Purgatory tests...");
let purgatory_results = specs::PurgatoryTests::run_all(&client).await;
all_results.merge(purgatory_results);
// Push authorization tests (mutates shared state - must run last among git specs)
println!(" → Push authorization tests...");
let push_results = specs::PushAuthorizationTests::run_all(&client, &relay_domain).await;
all_results.merge(push_results);
// GRASP-06 tests live in their own spec family with their
// own report renderer. Print the GRASP-01 block first
// (via the default `print_report` below), then the
// GRASP-06 block separately.
println!(" → GRASP-06 tests...");
let grasp06_results = specs::Grasp06Tests::run_all(&client).await;
println!();
all_results.print_report();
specs::Grasp06Tests::print_report(&grasp06_results);
let combined_ok =
all_results.all_passed() && grasp06_results.all_passed();
if !combined_ok {
println!("❌ Some tests failed");
std::process::exit(1);
} else {
println!("✅ All tests passed!");
}
return Ok(());
}
_ => {
return Err(anyhow!(
"Unknown spec: {}. Use 'nip01-smoke', 'nip11', 'event-acceptance', 'cors', 'git-clone', 'git-filter', 'push-auth', 'repo-creation', 'purgatory', 'grasp06', or 'all'",
spec
))
}
};
results.print_report();
if !results.all_passed() {
println!("❌ Some tests failed");
std::process::exit(1);
} else {
println!("✅ All tests passed!");
}
}
} => run_audit(relay, mode, spec, git_data_dir, json, keys).await?,
}
Ok(())
}
fn audit_json_requested(args: &[OsString]) -> bool {
args.get(1).is_some_and(|arg| arg == "audit") && args.iter().skip(2).any(|arg| arg == "--json")
}
fn requested_relay(args: &[OsString]) -> String {
for (index, arg) in args.iter().enumerate().skip(2) {
if arg == "--relay" || arg == "-r" {
return args
.get(index + 1)
.map(|value| value.to_string_lossy().into_owned())
.unwrap_or_default();
}
if let Some(value) = arg.to_str().and_then(|arg| arg.strip_prefix("--relay=")) {
return value.to_string();
}
}
String::new()
}
async fn run_audit(
relay: String,
mode: String,
spec: AuditSelection,
git_data_dir: Option<PathBuf>,
json: bool,
key_args: KeyArgs,
) -> Result<()> {
let started_at = unix_timestamp();
let started = Instant::now();
let mut config = match mode.as_str() {
"shared" | "production" => AuditConfig::shared(),
"isolated" | "ci" => AuditConfig::isolated(),
_ => audit_runtime_error(
json,
&relay,
"",
started_at,
started,
anyhow!("Invalid mode: {mode}. Use 'shared' or 'isolated'"),
),
};
config.read_only = false;
let run_id = config.run_id.clone();
let keys = match load_keys(&key_args) {
Ok(keys) => keys,
Err(error) => audit_runtime_error(json, &relay, &run_id, started_at, started, error),
};
let relay_domain = relay
.trim_start_matches("ws://")
.trim_start_matches("wss://")
.trim_end_matches('/')
.to_string();
if !json {
println!("🔍 GRASP Audit Tool");
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!("Relay: {relay}");
println!("Mode: {mode}");
println!("Spec: {spec}");
println!("Run ID: {run_id}");
if let Some(ref dir) = git_data_dir {
println!("Git Dir: {}", dir.display());
}
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n");
println!("Connecting to relay...");
}
let client_result = match keys {
Some(keys) => AuditClient::new_with_keys(&relay, config, keys).await,
None => AuditClient::new(&relay, config).await,
};
let client = match client_result {
Ok(client) => client,
Err(error) => audit_runtime_error(
json,
&relay,
&run_id,
started_at,
started,
anyhow!("Failed to connect to relay: {error}"),
),
};
if !client.is_connected().await {
audit_runtime_error(
json,
&relay,
&run_id,
started_at,
started,
anyhow!("Could not establish connection to relay"),
);
}
if !json {
println!("✓ Connected\n");
}
let mut results = Vec::new();
match spec {
AuditSelection::Nip01Smoke => {
status(json, "Running NIP-01 smoke tests...");
results.push((
AuditSuite::Nip01Smoke,
specs::Nip01SmokeTests::run_all(&client).await,
));
}
AuditSelection::Nip11 => {
status(json, "Running NIP-11 document tests...");
results.push((
AuditSuite::Nip11Document,
specs::Nip11DocumentTests::run_all(&client).await,
));
}
AuditSelection::EventAcceptance => {
status(json, "Running event acceptance policy tests...");
results.push((
AuditSuite::EventAcceptance,
specs::EventAcceptancePolicyTests::run_all(&client).await,
));
}
AuditSelection::Cors => {
status(json, "Running CORS tests...");
results.push((
AuditSuite::Cors,
specs::CorsTests::run_all(&client, &relay_domain).await,
));
}
AuditSelection::GitClone => {
status(json, "Running Git clone tests...");
results.push((
AuditSuite::GitClone,
specs::GitCloneTests::run_all(&client, &relay_domain).await,
));
}
AuditSelection::GitFilter => {
status(json, "Running Git filter capability tests...");
results.push((
AuditSuite::GitFilter,
specs::GitFilterTests::run_all(&client, &relay_domain).await,
));
}
AuditSelection::PushAuth => {
status(json, "Running push authorization tests...");
results.push((
AuditSuite::PushAuthorization,
specs::PushAuthorizationTests::run_all(&client, &relay_domain).await,
));
}
AuditSelection::RepoCreation => {
status(json, "Running repository creation tests...");
results.push((
AuditSuite::RepositoryCreation,
specs::RepositoryCreationTests::run_all(&client, &relay_domain).await,
));
}
AuditSelection::Purgatory => {
status(json, "Running purgatory tests...");
results.push((
AuditSuite::Purgatory,
specs::PurgatoryTests::run_all(&client).await,
));
}
AuditSelection::Grasp06 => {
status(json, "Running GRASP-06 tests...");
results.push((
AuditSuite::Grasp06All,
specs::Grasp06Tests::run_all(&client).await,
));
}
AuditSelection::All => {
status(json, "Running all tests...");
let mut grasp01 = AuditResult::new("All GRASP-01 Tests");
progress(json, " → NIP-01 smoke tests...");
grasp01.merge(specs::Nip01SmokeTests::run_all(&client).await);
progress(json, " → NIP-11 document tests...");
grasp01.merge(specs::Nip11DocumentTests::run_all(&client).await);
progress(json, " → CORS tests...");
grasp01.merge(specs::CorsTests::run_all(&client, &relay_domain).await);
progress(json, " → Repository creation tests...");
grasp01.merge(specs::RepositoryCreationTests::run_all(&client, &relay_domain).await);
progress(json, " → Git clone tests...");
grasp01.merge(specs::GitCloneTests::run_all(&client, &relay_domain).await);
progress(json, " → Git filter capability tests...");
grasp01.merge(specs::GitFilterTests::run_all(&client, &relay_domain).await);
progress(json, " → Event acceptance policy tests...");
grasp01.merge(specs::EventAcceptancePolicyTests::run_all(&client).await);
// Purgatory must run before push authorization because the latter
// replaces shared state events used by the purgatory assertions.
progress(json, " → Purgatory tests...");
grasp01.merge(specs::PurgatoryTests::run_all(&client).await);
progress(json, " → Push authorization tests...");
grasp01.merge(specs::PushAuthorizationTests::run_all(&client, &relay_domain).await);
results.push((AuditSuite::Grasp01All, grasp01));
progress(json, " → GRASP-06 tests...");
results.push((
AuditSuite::Grasp06All,
specs::Grasp06Tests::run_all(&client).await,
));
}
}
let all_passed = results.iter().all(|(_, result)| result.all_passed());
if json {
let specs = results
.iter()
.map(|(suite, result)| SpecReport::from_result(*suite, result))
.collect();
let report = AuditReport::completed(&relay, &run_id, started_at, started.elapsed(), specs);
println!("{}", serde_json::to_string(&report)?);
} else {
for (suite, result) in &results {
if *suite == AuditSuite::Grasp06All {
specs::Grasp06Tests::print_report(result);
} else {
result.print_report();
}
}
if all_passed {
println!("✅ All tests passed!");
} else {
println!("❌ Some tests failed");
}
}
if !all_passed {
std::process::exit(1);
}
Ok(())
}
fn status(json: bool, message: &str) {
if !json {
println!("{message}\n");
}
}
fn progress(json: bool, message: &str) {
if !json {
println!("{message}");
}
}
fn load_keys(args: &KeyArgs) -> Result<Option<Keys>> {
load_keys_with_env(args, std::env::var_os(GRASP_AUDIT_NSEC_ENV))
}
@@ -425,6 +506,32 @@ fn load_keys_with_env(args: &KeyArgs, env_value: Option<OsString>) -> Result<Opt
.map_err(|error| anyhow!("Invalid secret key from {source}: {error}"))
}
fn audit_runtime_error(
json: bool,
relay: &str,
run_id: &str,
started_at: u64,
started: Instant,
error: anyhow::Error,
) -> ! {
if json {
let report = AuditReport::error(
relay,
run_id,
started_at,
started.elapsed(),
error.to_string(),
);
println!(
"{}",
serde_json::to_string(&report).expect("audit error report is serializable")
);
} else {
eprintln!("{error:#}");
}
std::process::exit(2);
}
#[cfg(test)]
mod tests {
use super::*;
@@ -540,4 +647,34 @@ mod tests {
);
}
}
#[test]
fn audit_selection_is_validated_by_clap() {
let parsed = Cli::try_parse_from([
"grasp-audit",
"audit",
"--relay",
"wss://example.com",
"--spec",
"not-an-audit",
]);
assert!(parsed.is_err());
}
#[test]
fn json_parse_error_helpers_detect_audit_and_relay() {
let args: Vec<OsString> = [
"grasp-audit",
"audit",
"--json",
"--relay=wss://relay.example",
]
.into_iter()
.map(OsString::from)
.collect();
assert!(audit_json_requested(&args));
assert_eq!(requested_relay(&args), "wss://relay.example");
}
}
grasp-audit/src/lib.rs
+2 -1
View File
@@ -70,7 +70,8 @@ pub use probe::{
run_probe, run_probe_with_options, ProbeCheck, ProbeCheckName, ProbeOptions, ProbeReport,
};
pub use report::{
AuditOutcome, AuditSpec, AuditSuite, ResultCounts, SpecReport, TestOutcome, TestReport,
unix_timestamp, AuditOutcome, AuditReport, AuditRunOutcome, AuditSpec, AuditSuite,
ResultCounts, SpecReport, TestOutcome, TestReport, ToolInfo, AUDIT_REPORT_SCHEMA_VERSION,
};
pub use result::{AuditResult, TestResult};
grasp-audit/src/report.rs
+153 -5
View File
@@ -1,15 +1,17 @@
//! Stable, serializable results for compliance audit consumers.
//!
//! These types describe completed spec and test results. They deliberately do
//! not include invocation details such as the relay URL, run timestamps, or a
//! top-level schema version; a CLI machine-output contract can wrap them later
//! without changing the audit domain model.
//! [`SpecReport`] and [`TestReport`] describe completed audit results, while
//! [`AuditReport`] adds the versioned run metadata used by the CLI's machine
//! output contract.
use crate::result::{AuditResult, TestResult};
use crate::specs::grasp01::GRASP_COMMIT_ID;
use crate::specs::grasp06::GRASP_06_COMMIT_ID;
use serde::Serialize;
use std::time::Duration;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
/// Version of the full-audit JSON document emitted by the CLI.
pub const AUDIT_REPORT_SCHEMA_VERSION: u16 = 1;
/// Maximum UTF-8 byte length retained for failures and skip reasons.
pub const MAX_RESULT_MESSAGE_BYTES: usize = 4096;
@@ -91,6 +93,22 @@ pub enum AuditOutcome {
Fail,
}
/// Outcome of a full audit invocation.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum AuditRunOutcome {
Pass,
Fail,
Error,
}
/// Identity of the tool that produced a full audit report.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct ToolInfo {
pub name: &'static str,
pub version: &'static str,
}
/// Outcome of a single audit test.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
@@ -135,6 +153,74 @@ pub struct SpecReport {
pub tests: Vec<TestReport>,
}
/// Stable machine-readable envelope for one full audit CLI invocation.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct AuditReport {
pub schema_version: u16,
pub tool: ToolInfo,
pub relay_url: String,
pub run_id: String,
pub started_at: u64,
pub finished_at: u64,
pub duration_ms: u64,
pub outcome: AuditRunOutcome,
pub specs: Vec<SpecReport>,
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
}
impl AuditReport {
/// Build a report for an audit that reached test execution.
pub fn completed(
relay_url: impl Into<String>,
run_id: impl Into<String>,
started_at: u64,
duration: Duration,
specs: Vec<SpecReport>,
) -> Self {
let outcome = if specs.iter().all(|spec| spec.outcome == AuditOutcome::Pass) {
AuditRunOutcome::Pass
} else {
AuditRunOutcome::Fail
};
Self {
schema_version: AUDIT_REPORT_SCHEMA_VERSION,
tool: tool_info(),
relay_url: relay_url.into(),
run_id: run_id.into(),
started_at,
finished_at: unix_timestamp(),
duration_ms: duration_ms(duration),
outcome,
specs,
error: None,
}
}
/// Build a report for a parse, setup, or runtime error.
pub fn error(
relay_url: impl Into<String>,
run_id: impl Into<String>,
started_at: u64,
duration: Duration,
error: impl AsRef<str>,
) -> Self {
Self {
schema_version: AUDIT_REPORT_SCHEMA_VERSION,
tool: tool_info(),
relay_url: relay_url.into(),
run_id: run_id.into(),
started_at,
finished_at: unix_timestamp(),
duration_ms: duration_ms(duration),
outcome: AuditRunOutcome::Error,
specs: Vec::new(),
error: Some(bounded_message(error.as_ref())),
}
}
}
impl SpecReport {
/// Convert runtime results using the central metadata for `suite`.
pub fn from_result(suite: AuditSuite, result: &AuditResult) -> Self {
@@ -186,6 +272,21 @@ fn duration_ms(duration: Duration) -> u64 {
duration.as_millis().min(u64::MAX as u128) as u64
}
/// Current Unix timestamp in seconds.
pub fn unix_timestamp() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
fn tool_info() -> ToolInfo {
ToolInfo {
name: "grasp-audit",
version: env!("CARGO_PKG_VERSION"),
}
}
fn bounded_message(value: &str) -> String {
if value.len() <= MAX_RESULT_MESSAGE_BYTES {
return value.to_owned();
@@ -258,4 +359,51 @@ mod tests {
assert_eq!(TestReport::from(&result).duration_ms, 42);
}
#[test]
fn audit_report_wraps_stable_spec_reports() {
let mut passing = AuditResult::new("passing");
passing.add(TestResult::new("passes", SpecRef::NostrRelayNip01Compliant, "passes").pass());
let mut failing = AuditResult::new("failing");
failing.add(
TestResult::new("fails", SpecRef::NostrRelayNip01Compliant, "fails").fail("broken"),
);
let report = AuditReport::completed(
"wss://relay.example",
"audit-1",
1,
Duration::from_millis(42),
vec![
SpecReport::from_result(AuditSuite::Grasp01All, &passing),
SpecReport::from_result(AuditSuite::Grasp06All, &failing),
],
);
let json = serde_json::to_value(&report).unwrap();
assert_eq!(json["schema_version"], AUDIT_REPORT_SCHEMA_VERSION);
assert_eq!(json["tool"]["name"], "grasp-audit");
assert_eq!(json["relay_url"], "wss://relay.example");
assert_eq!(json["run_id"], "audit-1");
assert_eq!(json["duration_ms"], 42);
assert_eq!(json["outcome"], "fail");
assert_eq!(json["specs"][0]["suite"], "grasp01-all");
assert_eq!(json["specs"][1]["suite"], "grasp06-all");
assert!(json.get("error").is_none());
}
#[test]
fn audit_error_reports_are_bounded_and_have_no_specs() {
let report = AuditReport::error(
"wss://relay.example",
"audit-1",
1,
Duration::ZERO,
"é".repeat(MAX_RESULT_MESSAGE_BYTES),
);
assert_eq!(report.outcome, AuditRunOutcome::Error);
assert!(report.specs.is_empty());
assert!(report.error.unwrap().len() <= MAX_RESULT_MESSAGE_BYTES);
}
}
grasp-audit/tests/cli_json.rs
+81
View File
@@ -0,0 +1,81 @@
use std::process::Command;
fn run_invalid_audit(args: &[&str]) -> serde_json::Value {
let output = Command::new(env!("CARGO_BIN_EXE_grasp-audit"))
.args(args)
.output()
.unwrap();
assert_eq!(output.status.code(), Some(2));
assert!(
output.stderr.is_empty(),
"unexpected stderr: {}",
String::from_utf8_lossy(&output.stderr)
);
let stdout = String::from_utf8(output.stdout).unwrap();
assert_eq!(stdout.lines().count(), 1);
serde_json::from_str(&stdout).unwrap()
}
#[test]
fn missing_required_argument_uses_audit_json_schema() {
let report = run_invalid_audit(&["audit", "--json"]);
assert_eq!(report["schema_version"], 1);
assert_eq!(report["outcome"], "error");
assert_eq!(report["relay_url"], "");
assert_eq!(report["specs"], serde_json::json!([]));
assert!(report["error"]
.as_str()
.unwrap()
.contains("MissingRequiredArgument"));
}
#[test]
fn unknown_argument_uses_audit_json_schema() {
let report = run_invalid_audit(&[
"audit",
"--json",
"--relay",
"wss://relay.example",
"--unknown",
]);
assert_eq!(report["outcome"], "error");
assert_eq!(report["relay_url"], "wss://relay.example");
assert!(report["error"]
.as_str()
.unwrap()
.contains("UnknownArgument"));
}
#[test]
fn invalid_audit_spec_uses_audit_json_schema() {
let report = run_invalid_audit(&[
"audit",
"--json",
"--relay",
"wss://relay.example",
"--spec",
"not-an-audit",
]);
assert_eq!(report["outcome"], "error");
assert!(report["error"].as_str().unwrap().contains("InvalidValue"));
}
#[test]
fn invalid_mode_uses_audit_json_schema_without_connecting() {
let report = run_invalid_audit(&[
"audit",
"--json",
"--relay",
"wss://relay.example",
"--mode",
"invalid",
]);
assert_eq!(report["outcome"], "error");
assert_eq!(report["relay_url"], "wss://relay.example");
assert!(report["error"].as_str().unwrap().contains("Invalid mode"));
}