diff --git a/seeder-launcher/scripts/build-start9-s9pk.sh b/seeder-launcher/scripts/build-start9-s9pk.sh index cf93c33..ce7f284 100755 --- a/seeder-launcher/scripts/build-start9-s9pk.sh +++ b/seeder-launcher/scripts/build-start9-s9pk.sh @@ -6,15 +6,24 @@ # Start9 package to it, and builds + verifies the s9pk with a .sha256 sidecar in # seeder-launcher/start9/. # +# Emits TWO packages: the v1 s9pk that StartOS 0.3.5.x reads, and a v2 s9pk +# converted from it for 0.4.0+ (which refuses v1 in its web UI, and whose +# registry entry publishes a commitment computed over the v2 file). +# # Usage: build-start9-s9pk.sh [version] # # Version resolution: arg > release git tag (vX.Y.Z) > seeder package.json. # # Env: -# IMAGE base image repo (default ghcr.io/peerloomllc/pearcal-seeder) +# IMAGE base image repo (default ghcr.io/peerloomllc/pearcal-seeder) +# START_CLI_V2 0.4.x-era start-cli for the v2 conversion (default: found on PATH) +# START9_WORKSPACE packaging workspace holding the build signing key +# (default ~/.start9-workspace) # # Requires: the StartOS SDK (start-sdk), deno, yq, skopeo, and docker or podman -# (+ qemu-user-static for the arm64 image tar on an x86 host). +# (+ qemu-user-static for the arm64 image tar on an x86 host). The v2 conversion +# additionally needs start-cli 1.x plus a packaging workspace; without them it is +# skipped loudly rather than failing the build. set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" @@ -136,7 +145,82 @@ S9PK="$START9_DIR/pearcal-seeder.s9pk" ( cd "$START9_DIR" && sha256sum "pearcal-seeder.s9pk" > "pearcal-seeder.s9pk.sha256" ) echo "==> s9pk ready: $S9PK ($(du -h "$S9PK" | cut -f1))" +# --- also emit a v2 s9pk for StartOS 0.4.0+ -------------------------------- +# 0.4.0's web UI refuses a v1 package outright: sideload.utils.ts sniffs the +# magic bytes (3b 3b 01 vs 3b 3b 02) and tells the operator the format is +# deprecated. The OS still installs v1 through `start-cli package install +# --sideload`, so v1 is not dead - but "sideload from the browser" is how a +# StartOS user expects to install a package we do not list in a marketplace. +# +# It is also what the 0.4 REGISTRY entry is built from: the published +# `commitment` is computed over this file, so registry/build-registry-04.js +# needs it and 0.4 boxes download it (see the /package/v1 rule in the website's +# _redirects). +# +# We do NOT hand-author a second package for that. StartOS ships a converter +# (`start-cli s9pk convert`, backed by S9pk::from_v1), which is how Start9 +# migrated its own catalogue; a converted package keeps its 0.3.5-era +# procedures and gains " (Legacy)" on its title. So the v1 above stays the +# single source of truth and the v2 is derived from it. +# +# Needs the 0.4.x-era start-cli (the 0.3.5 SDK's `start-cli` cannot do this) +# and a packaging workspace, which holds the build signing key the converted +# package is signed with. Both are machine setup, not repo state - the key must +# never be committed. Create one with `start-cli s9pk init-workspace`. +V2_S9PK="$START9_DIR/pearcal-seeder-v2.s9pk" +START9_WORKSPACE="${START9_WORKSPACE:-$HOME/.start9-workspace}" + +# Resolve a start-cli that can convert. `start-cli --version` prints "StartOS +# CLI 0.3.5.1" for the old SDK and "start-cli 1.1.0" for the new one, so the +# leading token tells them apart without comparing version numbers. +_v2_cli="" +for _cand in "${START_CLI_V2:-}" start-cli-1.1.0 start-cli; do + [ -n "$_cand" ] || continue + command -v "$_cand" >/dev/null 2>&1 || continue + if "$_cand" --version 2>/dev/null | grep -qE '^start-cli [1-9]'; then _v2_cli="$_cand"; break; fi +done + +if [ -z "$_v2_cli" ] || [ ! -f "$START9_WORKSPACE/.startos/config.yaml" ]; then + echo "" + echo " !! SKIPPING the v2 s9pk - StartOS 0.4.0 users will not be able to" >&2 + echo " !! sideload this release from the web UI, and the 0.4 registry entry" >&2 + echo " !! cannot be regenerated (CLI sideload still works)." >&2 + # if/fi, not `[ ] && echo`: under `set -e` a false test would exit the script, + # turning a skipped optional artifact into a failed release build. + if [ -z "$_v2_cli" ]; then + echo " !! missing: a 0.4.x start-cli (set START_CLI_V2, or install start-cli 1.x)" >&2 + fi + if [ ! -f "$START9_WORKSPACE/.startos/config.yaml" ]; then + echo " !! missing: a packaging workspace at $START9_WORKSPACE" >&2 + fi + echo " !! fix: start-cli s9pk init-workspace $START9_WORKSPACE" >&2 + echo "" +else + echo "==> converting to a v2 s9pk for StartOS 0.4.0+ ($_v2_cli) ..." + # convert rewrites IN PLACE, so it operates on a copy - losing the v1 here + # would strand every 0.3.5 box. + cp -f "$S9PK" "$V2_S9PK" + # Run from inside the workspace: the converter walks up from the CWD looking + # for .startos, and signs with that workspace's build key. + if ( cd "$START9_WORKSPACE" && "$_v2_cli" s9pk convert "$V2_S9PK" ); then + # Trust the bytes, not the exit code: a v2 package starts 3b 3b 02. + if [ "$(head -c 3 "$V2_S9PK" | od -An -tx1 | tr -d ' \n')" = "3b3b02" ]; then + ( cd "$START9_DIR" && sha256sum "pearcal-seeder-v2.s9pk" > "pearcal-seeder-v2.s9pk.sha256" ) + echo "==> v2 s9pk ready: $V2_S9PK ($(du -h "$V2_S9PK" | cut -f1))" + else + rm -f "$V2_S9PK" + echo "build-start9-s9pk: conversion reported success but the output is not a v2 s9pk" >&2 + exit 1 + fi + else + rm -f "$V2_S9PK" + echo "build-start9-s9pk: v2 conversion failed" >&2 + exit 1 + fi +fi + echo "" echo "==> Done. Review + commit the pinned start9/ files with the release:" echo " $START9_DIR/{manifest.yaml,Dockerfile,scripts/procedures/migrations.ts}" echo "S9PK=$S9PK" +if [ -f "$V2_S9PK" ]; then echo "S9PK_V2=$V2_S9PK"; fi diff --git a/seeder-launcher/scripts/publish-start9-registry.sh b/seeder-launcher/scripts/publish-start9-registry.sh index ae2c3e4..2172d43 100644 --- a/seeder-launcher/scripts/publish-start9-registry.sh +++ b/seeder-launcher/scripts/publish-start9-registry.sh @@ -8,6 +8,14 @@ # release + s9pk asset exist — the metadata advertises the version and _redirects # points at that release's asset. # +# TWO registries are published, because StartOS changed protocol: +# 0.3.5.x the static /package/v0 tree (build-registry.sh) +# 0.4.0+ a single JSON-RPC payload at /registry/0.4/index.json, served by +# the website Worker at POST /rpc/v0 (build-registry-04.js) +# A 0.4 box reads NOTHING from the 0.3.5 tree, so skipping the second one makes +# the package invisible to every 0.4 user while looking fine on 0.3.5. Both +# generators upsert, since the registry is shared with the other PeerLoom seeders. +# # Usage: publish-start9-registry.sh [version] # # Env: @@ -19,6 +27,10 @@ # + print the git/gh commands to run yourself. # S9PK built s9pk path # (default seeder-launcher/start9/pearcal-seeder.s9pk) +# S9PK_V2 built v2 s9pk for the 0.4 registry +# (default seeder-launcher/start9/pearcal-seeder-v2.s9pk) +# START_CLI_V2 0.4.x-era start-cli, needed to inspect the v2 s9pk +# (default: found on PATH) # RELEASE_REPO GitHub owner/repo hosting the s9pk release asset # (default peerloomllc/pearcal-native) set -euo pipefail @@ -26,6 +38,10 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" START9_DIR="$REPO_ROOT/seeder-launcher/start9" RELEASE_REPO="${RELEASE_REPO:-peerloomllc/pearcal-native}" +# Registry paths touched by this script. Used for the leftover-clearing sweep, +# the has-anything-changed check and `git add`, so a new output path only has to +# be named once. +REG_PATHS=(package registry _redirects) # --- version --------------------------------------------------------------- VERSION="${1:-}" @@ -37,6 +53,17 @@ VERSION="${VERSION#v}" [ -d "$WEBSITE_DIR/.git" ] || { echo "publish-start9-registry: $WEBSITE_DIR is not a git clone" >&2; exit 1; } S9PK="${S9PK:-$START9_DIR/pearcal-seeder.s9pk}" [ -f "$S9PK" ] || { echo "publish-start9-registry: s9pk not found: $S9PK (run build-start9-s9pk.sh first)" >&2; exit 1; } +S9PK_V2="${S9PK_V2:-$START9_DIR/pearcal-seeder-v2.s9pk}" + +# The 0.4 payload is generated by inspecting the v2 s9pk, which needs the +# 0.4-era start-cli. Resolve it the same way build-start9-s9pk.sh does: the +# 0.3.5 SDK prints "StartOS CLI 0.3.5.1" and the new one "start-cli 1.1.0". +_v2_cli="" +for _cand in "${START_CLI_V2:-}" start-cli-1.1.0 start-cli; do + [ -n "$_cand" ] || continue + command -v "$_cand" >/dev/null 2>&1 || continue + if "$_cand" --version 2>/dev/null | grep -qE '^start-cli [1-9]'; then _v2_cli="$_cand"; break; fi +done AUTO="${WEBSITE_REGISTRY_PR:-}" BRANCH="start9-registry-pearcal-v${VERSION}" @@ -57,10 +84,10 @@ if [ "$AUTO" = 1 ]; then # revert a previous publish. It is pure generated output and is regenerated a # few lines down, so clear it. Scoped to the registry paths ONLY, so unrelated # edits elsewhere in the clone (CLAUDE.md, site content) are never touched. - if [ -n "$(git -C "$WEBSITE_DIR" status --porcelain -- package _redirects)" ]; then - echo " clearing leftover registry output from a previous run (package/, _redirects)" - git -C "$WEBSITE_DIR" checkout -q -- package _redirects 2>/dev/null || true - git -C "$WEBSITE_DIR" clean -qfd -- package 2>/dev/null || true + if [ -n "$(git -C "$WEBSITE_DIR" status --porcelain -- "${REG_PATHS[@]}")" ]; then + echo " clearing leftover registry output from a previous run (${REG_PATHS[*]})" + git -C "$WEBSITE_DIR" checkout -q -- "${REG_PATHS[@]}" 2>/dev/null || true + git -C "$WEBSITE_DIR" clean -qfd -- package registry 2>/dev/null || true fi git -C "$WEBSITE_DIR" checkout -q -B "$BRANCH" "origin/${base}" else @@ -97,21 +124,58 @@ else echo " appended _redirects rules for pearcal-seeder v$VERSION" fi +# --- the 0.4 registry ------------------------------------------------------ +# 0.4 boxes read none of the above. They POST to /rpc/v0, which the website +# Worker answers from this one generated payload. +if [ ! -f "$S9PK_V2" ] || [ -z "$_v2_cli" ]; then + echo "" + echo " !! SKIPPING the 0.4 registry entry - StartOS 0.4 users will not see" >&2 + echo " !! pearcal-seeder at all (0.3.5.x boxes are unaffected)." >&2 + if [ ! -f "$S9PK_V2" ]; then + echo " !! missing: the v2 s9pk at $S9PK_V2 (build-start9-s9pk.sh emits it)" >&2 + fi + if [ -z "$_v2_cli" ]; then + echo " !! missing: a 0.4.x start-cli (set START_CLI_V2, or install start-cli 1.x)" >&2 + fi + echo "" +else + echo "==> upserting pearcal-seeder into the 0.4 registry payload for v$VERSION ..." + mkdir -p "$WEBSITE_DIR/registry/0.4" + # 0.4 must download the V2 package: the published commitment is computed over + # it, so handing a 0.4 box the v1 file yields a hash it cannot match and the + # install fails. Hence its own /package/v1 path, leaving /package/v0 to serve + # 0.3.5 boxes the v1 file they can actually read. + if grep -qE '^//?package/v1/pearcal-seeder\.s9pk' "$RED"; then + sed -i -E "s#(${RELEASE_REPO}/releases/download/)v[0-9.]+(/pearcal-seeder-v2\.s9pk)#\1v${VERSION}\2#g" "$RED" + echo " bumped existing /package/v1 _redirects rules to v$VERSION" + else + { + echo "/package/v1/pearcal-seeder.s9pk ${ASSET_BASE}/v${VERSION}/pearcal-seeder-v2.s9pk 302" + echo "//package/v1/pearcal-seeder.s9pk ${ASSET_BASE}/v${VERSION}/pearcal-seeder-v2.s9pk 302" + } >> "$RED" + echo " appended /package/v1 _redirects rules for pearcal-seeder v$VERSION" + fi + START_CLI="$_v2_cli" node "$START9_DIR/registry/build-registry-04.js" "$S9PK_V2" \ + --url "https://peerloomllc.com/package/v1/pearcal-seeder.s9pk" \ + --icon "$START9_DIR/icon.png" \ + --out "$WEBSITE_DIR/registry/0.4/index.json" +fi + # Nothing to publish? (re-run at the same version — metadata is deterministic.) -if [ -z "$(git -C "$WEBSITE_DIR" status --porcelain -- package _redirects)" ]; then +if [ -z "$(git -C "$WEBSITE_DIR" status --porcelain -- "${REG_PATHS[@]}")" ]; then echo "==> registry already current for pearcal-seeder v$VERSION — nothing to publish." exit 0 fi if [ "$AUTO" = 1 ]; then ( cd "$WEBSITE_DIR" - git add package _redirects + git add "${REG_PATHS[@]}" git commit -q -m "chore: StartOS registry -> pearcal-seeder v${VERSION}" git push -q -f -u origin "$BRANCH" if [ -z "$(gh pr list --head "$BRANCH" --state open --json number --jq '.[0].number' 2>/dev/null)" ]; then gh pr create --base "$base" --head "$BRANCH" \ --title "chore: StartOS registry -> pearcal-seeder v${VERSION}" \ - --body "Upserted \`/package/v0\` metadata + \`_redirects\` for pearcal-seeder v${VERSION} (merged alongside the other listed packages). Auto-generated by publish-start9-registry.sh; merging deploys the registry (Cloudflare)." \ + --body "Upserted the \`/package/v0\` (StartOS 0.3.5.x) metadata and the \`/registry/0.4/index.json\` (StartOS 0.4+) payload, plus \`_redirects\`, for pearcal-seeder v${VERSION} — merged alongside the other listed packages. Auto-generated by publish-start9-registry.sh; merging deploys the registry (Cloudflare)." \ >/dev/null fi if gh pr merge "$BRANCH" --squash --delete-branch >/dev/null 2>&1; then @@ -124,7 +188,7 @@ else echo "" echo "==> Registry files regenerated in $WEBSITE_DIR (not committed). To publish:" echo " cd $WEBSITE_DIR" - echo " git checkout -B $BRANCH && git add package _redirects" + echo " git checkout -B $BRANCH && git add ${REG_PATHS[*]}" echo " git commit -m 'chore: StartOS registry -> pearcal-seeder v${VERSION}'" echo " git push -u origin $BRANCH" echo " gh pr create --base $base --fill && gh pr merge --squash --delete-branch" diff --git a/seeder-launcher/start9/registry/README.md b/seeder-launcher/start9/registry/README.md index 5d094d5..39d4fe9 100644 --- a/seeder-launcher/start9/registry/README.md +++ b/seeder-launcher/start9/registry/README.md @@ -10,6 +10,45 @@ database, no registry service, no signing keyring (the s9pk is self-signed by `start-sdk pack`; StartOS only checks the signature is internally valid). `build-registry.sh` generates that tree from the built `.s9pk`. +**StartOS 0.4 changed all of that** and reads none of the above — see +[Two registries](#two-registries-035x-and-04) below. Publishing only the 0.3.5 +tree leaves the package invisible on 0.4 while everything looks healthy on +0.3.5, which is exactly how it went unnoticed until 2026-07-27. + +## Two registries: 0.3.5.x and 0.4 + +| | StartOS 0.3.5.x | StartOS 0.4+ | +|---|---|---| +| Protocol | `GET /package/v0/...` static files | JSON-RPC over `POST /rpc/v0` | +| Served from | the static tree in this repo's output | `registry/0.4/index.json` on the website, dispatched by `worker.js` | +| Generated by | `build-registry.sh` | `build-registry-04.js` | +| Package format | v1 s9pk | **v2** s9pk (`start-cli s9pk convert`) | +| Download path | `/package/v0/pearcal-seeder.s9pk` | `/package/v1/pearcal-seeder.s9pk` | + +Both are published together by `publish-start9-registry.sh`, and both must stay +current: a 0.4 box ignores the static tree entirely, and a 0.3.5 box cannot read +the RPC endpoint. + +The two download paths are **not** interchangeable. The 0.4 entry publishes a +`commitment` computed over the v2 file, so serving 0.4 the v1 s9pk yields a hash +that cannot match and the install fails. + +What 0.4 demands that 0.3.5 did not, each learned by driving a real 0.4 client: + +- **`commitment` is mandatory** — omitting it fails deserialization. Read it + with `start-cli s9pk inspect commitment` (note the s9pk comes *before* + the subcommand, the opposite of most `start-cli` commands). +- **`signatures` must be non-empty or install fails** with "Invalid Signature + Signer(s) not accepted", even though browsing works fine. StartOS takes its + accept-policy from the asset's *own* signatures, so **self-signing is + sufficient** and an empty set can never be satisfied. See `signCommitment` in + `build-registry-04.js` for exactly what gets signed and why it uses `openssl` + rather than Node's `crypto`. +- **`icon` must be a real base64 data URL**; `null` is rejected. + +Requires the 0.4-era `start-cli` (1.x). The `start-cli` from the 0.3.5 SDK has +no `s9pk` subcommand at all; set `START_CLI_V2` if it is not first on `PATH`. + ## Combined (multi-package) registry PeerLoom serves **one** registry (`peerloomllc.com`) that lists several seeders @@ -20,8 +59,15 @@ merged — `index` (array; this id's entry is replaced in place), `latest` (`{id: version}`), and `info` (categories unioned); everything else (`manifest/`, `instructions/`, …) is namespaced by id. -> Every app publishing into the shared tree must use merge-aware tooling like -> this. A legacy `rm -rf package`-style generator would drop the other packages. +`build-registry-04.js` upserts the same way, and there it matters more: 0.4 +serves every package from a **single** document, so a generator that wrote that +file wholesale would delist every other app in one line. It replaces only this +id's entry under `packageIndex.packages`, unions the categories and leaves the +registry name alone. + +> Every app publishing into the shared registry must use merge-aware tooling +> like this. A legacy `rm -rf package`-style generator would drop the other +> packages. ## Generate @@ -37,6 +83,29 @@ The JSON shapes mirror the live `registry.start9.com` exactly: `icon` is raw base64 (no `data:` prefix), `instructions`/`license` are `/package/v0/...` paths, and each index entry embeds the normalized manifest. +For 0.4, generate the RPC payload from the **v2** s9pk instead: + +```bash +START_CLI=start-cli-1.1.0 node registry/build-registry-04.js \ + pearcal-seeder-v2.s9pk \ + --url https://peerloomllc.com/package/v1/pearcal-seeder.s9pk \ + --icon icon.png --out /path/to/website/registry/0.4/index.json +``` + +Test it against a real 0.4 client before publishing, rather than inferring from +the JSON. Serve the payload behind the same dispatch `worker.js` uses, then: + +```bash +start-cli-1.1.0 -r http://127.0.0.1:8099 registry info +start-cli-1.1.0 -r http://127.0.0.1:8099 registry package index +start-cli-1.1.0 -r http://127.0.0.1:8099 registry package get pearcal-seeder --format json +``` + +`registry package index` listing every expected id is the check that matters — +that is the exact call whose one-package answer made this package invisible. +Note `start-cli -r package install` sends `package.install` to the +*registry* and always fails; installing from a custom registry is a UI action. + ## Host it Serve the tree as a static site. **The protocol paths are extensionless, so @@ -57,17 +126,21 @@ for local testing): `node registry/serve-registry.js 8099 `. ### Deployed at peerloomllc.com (Cloudflare) The live registry is the PeerLoom website (a Cloudflare project that deploys on -merge to `main`). Its `_headers` sets the per-route Content-Types and -`_redirects` sends `/package/v0/pearcal-seeder.s9pk` to the GitHub Release asset -(the s9pk is hundreds of MiB, over Cloudflare's 25 MiB per-file limit). Registry +merge to `main`). Its `_headers` sets the per-route Content-Types, `worker.js` +answers `POST /rpc/v0` for 0.4 clients, and `_redirects` sends both +`/package/v0/pearcal-seeder.s9pk` (v1, for 0.3.5) and +`/package/v1/pearcal-seeder.s9pk` (v2, for 0.4) to the GitHub Release assets — +the s9pks are hundreds of MiB, over Cloudflare's 25 MiB per-file limit. Registry URL users add: `https://peerloomllc.com`. On a seeder release this is automated by the release pipeline: -`build-start9-s9pk.sh` builds the s9pk (uploaded to the release tag by +`build-start9-s9pk.sh` builds both s9pks (uploaded to the release tag by `release.sh`), then `seeder-launcher/scripts/publish-start9-registry.sh` -upserts the metadata, points `_redirects` at the new tag, and (with -`WEBSITE_REGISTRY_PR=1`) opens + squash-merges the website PR — the merge is the -deploy. +upserts the 0.3.5 metadata *and* the 0.4 payload, points `_redirects` at the new +tag, and (with `WEBSITE_REGISTRY_PR=1`) opens + squash-merges the website PR — +the merge is the deploy. If the v2 s9pk or a 1.x `start-cli` is missing the 0.4 +half is skipped with a loud warning rather than failing the release, so watch +for that warning: it means 0.4 users will not see the new version. To refresh it by hand against a website clone: diff --git a/seeder-launcher/start9/registry/build-registry-04.js b/seeder-launcher/start9/registry/build-registry-04.js new file mode 100755 index 0000000..5f19f28 --- /dev/null +++ b/seeder-launcher/start9/registry/build-registry-04.js @@ -0,0 +1,276 @@ +#!/usr/bin/env node +// Emit the StartOS 0.4 registry payload for a v2 s9pk. +// +// StartOS 0.4 marketplaces do not read the 0.3.5 static tree that +// build-registry.sh produces. They speak JSON-RPC over POST /rpc/v0, so +// `https://peerloomllc.com` fails on 0.4 with "RPC ERROR: Network Error Not +// Found" while still working perfectly on 0.3.5.x boxes. +// +// The whole payload is static per release, so it is generated here and served +// verbatim by a tiny Worker. That keeps the request path dumb: no s9pk +// inspection, no crypto, no start-cli at the edge. +// +// COMBINED registry: like build-registry.sh for the 0.3.5 tree, this UPSERTS +// this package into whatever payload already exists at --out — it does NOT +// write the file wholesale. peerloomllc.com is one registry listing several +// PeerLoom seeders (pearcal-seeder, pearcircle-seeder, ...), and 0.4 serves +// them from a SINGLE generated document, so a generator that overwrote it +// would silently delist every other app. Only this id's entry under +// packageIndex.packages is replaced; categories are unioned and the registry +// name is left alone. +// +// NOTE: every app publishing into the shared payload must use merge-aware +// tooling like this. Keep this file in step with the sibling copies in the +// other PeerLoom repos — they are deliberately near-identical. +// +// What 0.4 actually demands, established by driving a real 0.4 client +// (`start-cli -r registry package index`) against a stub registry on +// 2026-07-27: +// - `commitment` is MANDATORY. Omitting it fails deserialization outright. +// Its value comes from the s9pk itself, so we can generate it. +// - `icon` must be a real base64 data URL. `null` is rejected. +// - `signatures` may be empty ONLY to browse. An unsigned entry lists fine +// and then fails install with "Invalid Signature Signer(s) not accepted". +// Self-signing is enough - see signCommitment below for why, and for what +// exactly gets signed. +// +// Usage: +// build-registry-04.js --url \ +// [--icon ] [--out ] [--registry-name ] [--signing-key ] +// +// Env: START_CLI overrides the start-cli binary. It must be the 0.4-era one +// (1.x); the 0.3.5 SDK's start-cli has no `s9pk inspect` and will fail. + +const { execFileSync } = require('child_process') +const fs = require('fs') +const os = require('os') +const path = require('path') + +function arg (flag, fallback = null) { + const i = process.argv.indexOf(flag) + return i > -1 && process.argv[i + 1] ? process.argv[i + 1] : fallback +} + +const S9PK = process.argv[2] +if (!S9PK || S9PK.startsWith('--')) { + console.error('usage: build-registry-04.js --url [--icon f.png] [--out f.json]') + process.exit(1) +} +if (!fs.existsSync(S9PK)) { + console.error(`build-registry-04: s9pk not found: ${S9PK}`) + process.exit(1) +} + +const S9PK_URL = arg('--url') +if (!S9PK_URL) { + console.error('build-registry-04: --url is required (where the s9pk is downloadable)') + process.exit(1) +} + +const START_CLI = process.env.START_CLI || 'start-cli' +const ICON = arg('--icon', path.join(path.dirname(S9PK), 'icon.png')) +const OUT = arg('--out', path.join(path.dirname(S9PK), 'registry-04.json')) +const REGISTRY_NAME = arg('--registry-name', 'PeerLoom Registry') +// The key the entry is self-signed with. Defaults to the same developer key +// that signs the s9pk itself, so the registry and the package share an identity. +const SIGNING_KEY = arg('--signing-key', process.env.START9_SIGNING_KEY || `${process.env.HOME}/.embassy/developer.key.pem`) + +// Categories are the registry's own vocabulary; package entries reference +// these keys. Kept in step with the 0.3.5 tree so a package does not appear +// under different headings depending on which StartOS version is asking. +const CATEGORIES = { + featured: { name: 'Featured' }, + networking: { name: 'Networking' }, +} +const PACKAGE_CATEGORIES = ['featured', 'networking'] + +// `inspect` takes the s9pk BEFORE the subcommand (`inspect manifest`), +// which is the opposite of most start-cli commands and an easy hour to lose. +function inspect (sub) { + const raw = execFileSync(START_CLI, ['s9pk', 'inspect', S9PK, sub], { + encoding: 'utf8', + maxBuffer: 64 * 1024 * 1024, + }) + return JSON.parse(raw) +} + +// 0.3.5 manifests carry plain strings where 0.4 wants a locale map. Only +// en_US is claimed: asserting a translation we do not have would be worse +// than offering one language. +const loc = (v) => (typeof v === 'string' && v.length > 0 ? { en_US: v } : { en_US: '' }) + +// Sign the commitment so the entry can actually be INSTALLED, not just browsed. +// +// An unsigned entry lists fine and then fails install with "Invalid Signature +// Signer(s) not accepted". StartOS derives its accept-policy from the asset's +// OWN signatures - RegistryAsset::all_signers() returns +// AcceptSigners::All(signatures.keys()), and install/mod.rs calls +// `asset.validate(SIG_CONTEXT, asset.all_signers())`. So self-signing is +// sufficient: any key is accepted provided its signature verifies. With no +// signatures the policy is All([]), which never becomes Accepted - hence the +// error, permanently, whatever the key. +// +// What gets signed (sign/commitment/merkle_archive.rs): SHA-512 over the raw +// root_sighash bytes followed by root_maxsize as a big-endian u64. Signed with +// Ed25519ph - the PREHASHED variant - under context "s9pk" (s9pk/v2/mod.rs). +// +// openssl does this correctly and is used rather than a JS library because +// neither Node's crypto nor Python's cryptography exposes Ed25519ph with a +// context; both would silently produce a plain-Ed25519 signature that looks +// right and fails verification. Validated against the RFC 8032 section 7.3 +// Ed25519ph test vector, which this openssl reproduces byte for byte. +function signCommitment (c) { + const msg = Buffer.concat([ + Buffer.from(c.rootSighash, 'base64url'), + (() => { const b = Buffer.alloc(8); b.writeBigUInt64BE(BigInt(c.rootMaxsize)); return b })(), + ]) + + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 's9sig-')) + const msgFile = path.join(tmp, 'msg.bin') + const sigFile = path.join(tmp, 'sig.bin') + const pubFile = path.join(tmp, 'pub.pem') + const ph = ['-pkeyopt', 'instance:ed25519ph', '-pkeyopt', 'context-string:s9pk'] + + try { + fs.writeFileSync(msgFile, msg) + + execFileSync('openssl', ['pkey', '-in', SIGNING_KEY, '-pubout', '-out', pubFile], + { stdio: ['ignore', 'ignore', 'pipe'] }) + execFileSync('openssl', + ['pkeyutl', '-sign', '-inkey', SIGNING_KEY, '-rawin', '-in', msgFile, ...ph, '-out', sigFile], + { stdio: ['ignore', 'ignore', 'pipe'] }) + + // Fail closed. An entry whose signature does not verify would list happily + // and then refuse to install, which is worse than not publishing at all. + execFileSync('openssl', + ['pkeyutl', '-verify', '-pubin', '-inkey', pubFile, '-rawin', '-in', msgFile, + '-sigfile', sigFile, ...ph], + { stdio: ['ignore', 'ignore', 'pipe'] }) + + // Registry format, matching registry.start9.com: the key is a standard SPKI + // PEM, and the value is the 64-byte signature DER-wrapped with the Ed25519 + // algorithm identifier (SEQUENCE { AlgorithmIdentifier, OCTET STRING }) in a + // PEM block labelled SIGNATURE. + const rawSig = fs.readFileSync(sigFile) + if (rawSig.length !== 64) throw new Error(`expected a 64-byte signature, got ${rawSig.length}`) + const der = Buffer.concat([Buffer.from('3049300506032b65700440', 'hex'), rawSig]) + const sigPem = '-----BEGIN SIGNATURE-----\n' + + (der.toString('base64').match(/.{1,64}/g) ?? []).join('\n') + + '\n-----END SIGNATURE-----\n' + + return { [fs.readFileSync(pubFile, 'utf8')]: sigPem } + } finally { + fs.rmSync(tmp, { recursive: true, force: true }) + } +} + +console.error(`==> inspecting ${path.basename(S9PK)} (this reads the whole package, give it a minute)`) +const manifest = inspect('manifest') +const commitment = inspect('commitment') + +if (!fs.existsSync(ICON)) { + console.error(`build-registry-04: icon not found: ${ICON} (0.4 rejects a null icon)`) + process.exit(1) +} +const iconDataUrl = 'data:image/png;base64,' + fs.readFileSync(ICON).toString('base64') + +const version = manifest.version // already "1.0.35:0" shape in a v2 manifest +if (!version) { + console.error('build-registry-04: manifest has no version') + process.exit(1) +} + +const arches = manifest.hardwareRequirements?.arch ?? ['x86_64', 'aarch64'] + +// --- read whatever is already published ------------------------------------ +// Missing or unreadable is normal (first run). Anything else present is another +// package's entry and must survive. +let prior = null +try { + prior = JSON.parse(fs.readFileSync(OUT, 'utf8')) +} catch { + prior = null +} +const priorPackages = prior?.packageIndex?.packages ?? {} + +// Re-stamping publishedAt on every run would churn the payload even when +// nothing about the package changed, and publish-start9-registry.sh decides +// whether there is anything to publish by diffing the working tree. So carry +// the previous timestamp forward when this exact package is already published +// at the same version and commitment, and stamp fresh otherwise. +const priorAsset = priorPackages[manifest.id]?.versions?.[version]?.s9pks?.[0]?.[1] +const unchanged = priorAsset && + priorAsset.commitment?.rootSighash === commitment.rootSighash && + String(priorAsset.commitment?.rootMaxsize) === String(commitment.rootMaxsize) && + Array.isArray(priorAsset.urls) && priorAsset.urls[0] === S9PK_URL +const publishedAt = arg('--published-at', + unchanged ? priorAsset.publishedAt : new Date().toISOString().replace('Z', '000000Z')) + +const versionEntry = { + title: manifest.title, + description: { + short: loc(manifest.description?.short), + long: loc(manifest.description?.long), + }, + releaseNotes: loc(manifest.releaseNotes), + // start-cli appends a newline (and "-modified" for a dirty tree) to gitHash. + gitHash: typeof manifest.gitHash === 'string' ? manifest.gitHash.trim() : null, + license: manifest.license ?? null, + packageRepo: manifest.packageRepo ?? null, + upstreamRepo: manifest.upstreamRepo ?? null, + marketingUrl: manifest.marketingUrl ?? null, + donationUrl: manifest.donationUrl ?? null, + osVersion: manifest.osVersion ?? null, + sdkVersion: manifest.sdkVersion ?? null, + hardwareAcceleration: manifest.hardwareAcceleration ?? false, + userspaceFilesystems: manifest.userspaceFilesystems ?? false, + virtualNetworking: manifest.virtualNetworking ?? false, + plugins: manifest.plugins ?? [], + satisfies: manifest.satisfies ?? [], + icon: iconDataUrl, + dependencyMetadata: {}, + sourceVersion: null, + // One entry: a hardware predicate paired with where to get the file. + s9pks: [[ + { device: [], ram: null, arch: arches }, + { + publishedAt, + urls: [S9PK_URL], + commitment, + signatures: signCommitment(commitment), + }, + ]], +} + +// --- UPSERT this package, preserving every other one ----------------------- +// We publish a single version per package, so this id's versions map is +// replaced rather than accumulated; other ids are copied through untouched. +const packages = { ...priorPackages } +packages[manifest.id] = { + authorized: {}, + categories: PACKAGE_CATEGORIES, + versions: { [version]: versionEntry }, +} +// Stable order by id so re-runs produce a deterministic diff. +const orderedPackages = Object.fromEntries( + Object.entries(packages).sort(([a], [b]) => a.localeCompare(b)), +) + +// Categories are unioned: another package may list under headings we do not. +const categories = { ...(prior?.packageIndex?.categories ?? {}), ...(prior?.info?.categories ?? {}), ...CATEGORIES } + +const payload = { + // The registry name belongs to the registry, not to any one package, so an + // existing one wins - otherwise two apps with different defaults would flip + // it back and forth on alternate releases. + info: { name: prior?.info?.name ?? REGISTRY_NAME, icon: prior?.info?.icon ?? null, categories }, + packageIndex: { + categories, + packages: orderedPackages, + }, +} + +fs.writeFileSync(OUT, JSON.stringify(payload, null, 2)) +console.error(`==> wrote ${OUT}`) +console.error(` ${manifest.id} ${version} | arch ${arches.join(',')} | rootSighash ${commitment.rootSighash}`) +console.error(` payload lists ${Object.keys(orderedPackages).length} package(s): ${Object.keys(orderedPackages).join(', ')}`)