141 lines
4.8 KiB
Python
Executable File
141 lines
4.8 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import datetime as dt
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from tests.harness.agent_process import AgentProcess
|
|
from tests.harness.didactyl_client import DidactylClient
|
|
from tests.harness.log_watcher import LogWatcher
|
|
from tests.harness.reporter import Reporter
|
|
from tests.harness.test_runner import TestRunner
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description="Didactyl automated test harness")
|
|
parser.add_argument("--config", default="tests/configs/test_genesis.jsonc")
|
|
parser.add_argument("--binary", default=None, help="Path to didactyl binary; if omitted, build_static --debug is used")
|
|
parser.add_argument("--suite", action="append", help="Run only this suite (repeatable)")
|
|
parser.add_argument("--test", action="append", help="Run only this test name (repeatable)")
|
|
parser.add_argument("--api-port", type=int, default=8485)
|
|
parser.add_argument("--api-bind", default="127.0.0.1")
|
|
parser.add_argument("--timeout", type=float, default=60.0)
|
|
parser.add_argument("--debug-level", type=int, default=5)
|
|
parser.add_argument("--output-dir", default=None)
|
|
parser.add_argument("--verbose", action="store_true")
|
|
parser.add_argument("--no-restart", action="store_true")
|
|
parser.add_argument(
|
|
"--skip-build-static-debug",
|
|
action="store_true",
|
|
help="Do not invoke ./build_static.sh --debug before running",
|
|
)
|
|
return parser.parse_args()
|
|
|
|
|
|
def build_static_debug_binary() -> None:
|
|
cmd = ["./build_static.sh", "--debug"]
|
|
print("Building debug static binary via ./build_static.sh --debug ...")
|
|
subprocess.run(cmd, check=True)
|
|
|
|
|
|
def resolve_default_debug_binary() -> Path:
|
|
candidates = [
|
|
Path("./didactyl_static_x86_64_debug"),
|
|
Path("./didactyl_static_arm64_debug"),
|
|
Path("./didactyl_static_armv7_debug"),
|
|
Path("./didactyl_static_armv6_debug"),
|
|
]
|
|
existing = [p for p in candidates if p.exists() and p.is_file()]
|
|
if existing:
|
|
return existing[0]
|
|
return Path("./didactyl_static_x86_64_debug")
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
|
|
run_ts = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
|
output_dir = args.output_dir or f"tests/results/{run_ts}"
|
|
Path(output_dir).mkdir(parents=True, exist_ok=True)
|
|
|
|
if not args.skip_build_static_debug:
|
|
try:
|
|
build_static_debug_binary()
|
|
except subprocess.CalledProcessError as exc:
|
|
print(f"build_static debug build failed with exit code {exc.returncode}")
|
|
return 2
|
|
|
|
binary = Path(args.binary) if args.binary else resolve_default_debug_binary()
|
|
if not binary.exists():
|
|
print(f"didactyl binary not found: {binary}")
|
|
print("Pass --binary <path> or run ./build_static.sh --debug")
|
|
return 2
|
|
|
|
config = Path(args.config)
|
|
if not config.exists():
|
|
print(f"config not found: {config}")
|
|
return 2
|
|
|
|
log_file = str(Path(output_dir) / "agent_debug.log")
|
|
base_url = f"https://{args.api_bind}:{args.api_port}"
|
|
|
|
agent = AgentProcess(
|
|
binary_path=str(binary),
|
|
config_path=str(config),
|
|
api_port=args.api_port,
|
|
api_bind=args.api_bind,
|
|
debug_level=args.debug_level,
|
|
log_file=log_file,
|
|
base_url=base_url,
|
|
)
|
|
client = DidactylClient(base_url=base_url, timeout=args.timeout, verify_tls=False)
|
|
log = LogWatcher(log_file)
|
|
|
|
log.start()
|
|
started = agent.start(timeout=30)
|
|
if not started:
|
|
print("Failed to start Didactyl agent")
|
|
log.stop()
|
|
return 1
|
|
|
|
try:
|
|
runner = TestRunner(agent=agent, client=client, log=log, args=args)
|
|
tests = runner.discover_suites("tests.suites")
|
|
if not tests:
|
|
print("No tests discovered")
|
|
return 3
|
|
|
|
results = runner.run_all(tests)
|
|
|
|
run_meta = {
|
|
"run_timestamp": dt.datetime.now(dt.timezone.utc).isoformat(),
|
|
"base_url": base_url,
|
|
"config": str(config),
|
|
"binary": str(binary),
|
|
"test_count": len(results),
|
|
}
|
|
reporter = Reporter(results=results, run_meta=run_meta, output_dir=output_dir)
|
|
reporter.print_summary()
|
|
reporter.print_details()
|
|
json_path = reporter.write_json("results.json")
|
|
txt_path = reporter.write_text("results.txt")
|
|
print(f"\nWrote: {json_path}")
|
|
print(f"Wrote: {txt_path}")
|
|
|
|
bad = [r for r in results if r.status in {"fail", "error", "timeout"}]
|
|
return 1 if bad else 0
|
|
finally:
|
|
agent.stop(timeout=10)
|
|
log.stop()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|