Send publish response after outbox relays complete, then run broadcast batches in background — composer clears immediately while broadcast continues

plans/broadcast-batching-fix.md

@@ -0,0 +1,122 @@
+# Broadcast Relay Batching Fix
+
+## Problem
+
+With 600+ broadcast relays, NDK's `fromRelayUrls` creates 630 temporary relays
+and adds them all to the pool simultaneously. `NDKRelaySet.publish()` then fires
+all 630 relay publish operations in parallel via `Promise.all`. Each relay that
+isn't connected calls `relay.connect()` which opens a WebSocket.
+
+Browsers limit simultaneous WebSocket connections (practically ~50-100 at a
+time before connections queue up). With 630 simultaneous connection attempts,
+most relays never get a connection slot before the per-relay timeout expires.
+Result: only ~13/630 relays succeed.
+
+## Solution: Chunked publishing
+
+Instead of creating one giant `NDKRelaySet` with all 630 relays and calling
+`publish()` once, we split the relay URLs into **batches of 50** and publish
+to each batch sequentially. Each batch gets its own `NDKRelaySet.fromRelayUrls()`
+call, its own `publish()`, and its own set of live progress listeners.
+
+```mermaid
+flowchart TD
+  A[630 relay URLs] --> B[Split into batches of 50]
+  B --> C[Batch 1: 50 relays]
+  C --> D[publish to batch 1]
+  D --> E{Batch 1 done?}
+  E -->|yes| F[Batch 2: 50 relays]
+  F --> G[publish to batch 2]
+  G --> H{Batch 2 done?}
+  H -->|yes| I[... continue ...]
+  I --> J[Batch 13: 30 relays]
+  J --> K[publish to batch 13]
+  K --> L{All batches done?}
+  L -->|yes| M[Final done event with total count]
+```
+
+### Key design points
+
+1. **Batch size: 50 relays** — small enough to stay within browser WebSocket
+   limits, large enough to finish in a reasonable time. Each batch takes ~5-10s.
+
+2. **Sequential batches** — wait for each batch to complete before starting the
+   next. This keeps the total simultaneous connections at ~50, not 630.
+
+3. **Live progress across batches** — the `publishedSoFar` and `failedSoFar`
+   sets accumulate across all batches. The `broadcastProgress` events stream
+   continuously, so the footer shows the count climbing from 0 to 630 across
+   all batches.
+
+4. **Per-relay timeout: 10s per batch** — each relay in a batch gets 10s to
+   connect + publish. Since only 50 are competing for connections (not 630),
+   this is plenty of time.
+
+5. **Overall deadline: removed** — since batches are sequential and each has
+   its own per-relay timeout, the overall time is naturally bounded at
+   `numBatches × perBatchTimeout`. No need for a separate overall deadline.
+
+6. **Connection timeout: 10s** — set on each relay in the batch, matching the
+   publish timeout.
+
+## Implementation (in `www/ndk-worker.js` `handlePublish`)
+
+Replace the current single `NDKRelaySet.fromRelayUrls(allUrls, ndk)` + single
+`publish()` with a batched loop:
+
+```js
+const BATCH_SIZE = 50;
+const PER_RELAY_TIMEOUT_MS = 10000;
+
+if (isBroadcast) {
+    const allUrls = [...new Set([...outboxWriteUrls, ...activeBroadcastUrls])];
+    const batches = [];
+    for (let i = 0; i < allUrls.length; i += BATCH_SIZE) {
+        batches.push(allUrls.slice(i, i + BATCH_SIZE));
+    }
+    console.log(`[Worker] Broadcasting to ${allUrls.length} relays in ${batches.length} batches of ${BATCH_SIZE}`);
+
+    // Emit start event
+    broadcast({ type: 'broadcastProgress', sessionId, eventId, eventKind, phase: 'start', total: allUrls.length, successful: 0, failed: 0, latestRelay: null });
+
+    // Attach listeners once — they accumulate across all batches
+    ndkEvent.on('relay:published', (relay) => { ... publishedSoFar.add ... });
+    ndkEvent.on('relay:publish:failed', (relay, err) => { ... failedSoFar.add ... });
+
+    // Publish to each batch sequentially
+    for (let i = 0; i < batches.length; i++) {
+        const batch = batches[i];
+        const batchSet = NDKRelaySet.fromRelayUrls(batch, ndk);
+        // Set connection timeout on each relay in this batch
+        for (const relay of batchSet.relays) {
+            try { relay.connectionTimeout = PER_RELAY_TIMEOUT_MS; } catch (_) {}
+        }
+        try {
+            await ndkEvent.publish(batchSet, PER_RELAY_TIMEOUT_MS, 1);
+        } catch (e) {
+            // Expected — some relays in this batch may fail
+        }
+        console.log(`[Worker] Batch ${i+1}/${batches.length} done: ${publishedSoFar.size}/${allUrls.length} total succeeded`);
+    }
+
+    // Emit final done event
+    broadcast({ type: 'broadcastProgress', sessionId, eventId, eventKind, phase: 'done', total: allUrls.length, successful: publishedSoFar.size, failed: failedSoFar.size + timedOutSoFar.size, relayUrls: Array.from(publishedSoFar) });
+    // Clean up listeners
+    ndkEvent.removeAllListeners('relay:published');
+    ndkEvent.removeAllListeners('relay:publish:failed');
+}
+```
+
+## Files touched
+
+| File | Change |
+|------|--------|
+| [`www/ndk-worker.js`](www/ndk-worker.js) | Replace single publish with batched loop in `handlePublish` |
+
+## Testing
+
+1. Post with a user that has 600+ broadcast relays
+2. Watch the footer: count should climb steadily as batches complete
+3. Final count should be much higher than 13 (ideally 500+)
+4. Each batch of 50 should take ~5-10 seconds
+5. Total time for 630 relays: ~13 batches × ~10s = ~130 seconds

www/js/init-ndk.mjs

@@ -392,6 +392,7 @@ function handleWorkerMessage(event) {
                 failed: message.failed,
                 latestRelay: message.latestRelay,
                 latestError: message.latestError,
+                relayUrls: message.relayUrls || [],
             }
         }));
     } else if (message.type === 'startupStatus') {

www/js/post-interactions.mjs

@@ -915,16 +915,78 @@ export function formatTimeAgo(timestamp) {
 }
 
 // Store timestamps for real-time updates
-const timeAgoElements = new Map(); // element -> timestamp
+const timeAgoElements = new Map(); // element -> { timestamp, eventId }
+
+// Track relay publish info per event ID from broadcastProgress events.
+// Keyed by event ID → { count, urls } where urls is an array of relay URLs.
+const relayInfoByEventId = new Map();
+
+// Listen for broadcast progress events to capture final relay counts and URLs.
+// The worker emits ndkBroadcastProgress with phase 'done' containing the
+// total successful count and the list of relay URLs. We store it so post
+// cards can show "21r - 1h" with a hover tooltip listing the relays.
+if (typeof window !== 'undefined') {
+    window.addEventListener('ndkBroadcastProgress', (event) => {
+        const d = event.detail;
+        if (!d || d.phase !== 'done') return;
+        const eventId = d.eventId;
+        if (!eventId) return;
+        relayInfoByEventId.set(eventId, {
+            count: d.successful || 0,
+            urls: Array.isArray(d.relayUrls) ? d.relayUrls : [],
+        });
+        // Update any already-rendered time elements for this event.
+        timeAgoElements.forEach((info, element) => {
+            if (info.eventId === eventId && element.isConnected) {
+                updateTimeAgoElement(element, info);
+            }
+        });
+    });
+}
+
+/**
+ * Build the hover tooltip text for a relay list.
+ * Shows "Published to N relays:" followed by the URL list, truncated.
+ * @param {number} count - Number of successful relays
+ * @param {string[]} urls - Array of relay URLs
+ * @returns {string}
+ */
+function buildRelayTooltip(count, urls) {
+    if (!count || count === 0) return '';
+    const header = `Published to ${count} relay${count === 1 ? '' : 's'}:`;
+    if (!urls || urls.length === 0) return header;
+    // Show up to 50 relay URLs in the tooltip; beyond that, show a summary.
+    if (urls.length <= 50) {
+        return header + '\n' + urls.join('\n');
+    }
+    return header + '\n' + urls.slice(0, 50).join('\n') + `\n… and ${urls.length - 50} more`;
+}
+
+/**
+ * Update a time-ago element's text and tooltip from its stored info.
+ * @param {HTMLElement} element - The element to update
+ * @param {{timestamp: number, eventId: string}} info - Stored info
+ */
+function updateTimeAgoElement(element, info) {
+    const timeStr = formatTimeAgo(info.timestamp);
+    if (info.eventId && relayInfoByEventId.has(info.eventId)) {
+        const relayInfo = relayInfoByEventId.get(info.eventId);
+        element.textContent = `${relayInfo.count}r - ${timeStr}`;
+        element.title = buildRelayTooltip(relayInfo.count, relayInfo.urls);
+    } else {
+        element.textContent = timeStr;
+    }
+}
 
 /**
  * Register an element for time updates
  * @param {HTMLElement} element - The element to update
  * @param {number} timestamp - Unix timestamp in seconds
+ * @param {string} [eventId] - Optional event ID for relay count display
  */
-export function registerTimeAgo(element, timestamp) {
-    timeAgoElements.set(element, timestamp);
-    element.textContent = formatTimeAgo(timestamp);
+export function registerTimeAgo(element, timestamp, eventId) {
+    timeAgoElements.set(element, { timestamp, eventId });
+    updateTimeAgoElement(element, { timestamp, eventId });
 }
 
 /**
@@ -932,9 +994,9 @@ export function registerTimeAgo(element, timestamp) {
  * Call this periodically (e.g., every 30 seconds)
  */
 export function updateTimeAgos() {
-    timeAgoElements.forEach((timestamp, element) => {
+    timeAgoElements.forEach((info, element) => {
         if (element.isConnected) {
-            element.textContent = formatTimeAgo(timestamp);
+            updateTimeAgoElement(element, info);
         } else {
             // Clean up detached elements
             timeAgoElements.delete(element);
@@ -1085,10 +1147,12 @@ export function renderFooterRow(eventId, eventData, options = {}) {
     container.appendChild(nutzapItem);
     container.appendChild(midControls);
 
-    // Time as the final item in the same row
+    // Time as the final item in the same row.
+    // Pass eventId so registerTimeAgo can prepend the relay count
+    // (e.g., "21r - 1h") when a broadcastProgress 'done' event arrives.
     const timeEl = document.createElement('span');
     timeEl.className = 'divPostTime';
-    registerTimeAgo(timeEl, eventData.created_at);
+    registerTimeAgo(timeEl, eventData.created_at, eventId);
     container.appendChild(timeEl);
 
     footerRow.appendChild(container);

www/js/version.json

@@ -1,5 +1,5 @@
 {
-  "VERSION": "v0.7.69",
-  "VERSION_NUMBER": "0.7.69",
-  "BUILD_DATE": "2026-06-30T13:19:30.001Z"
+  "VERSION": "v0.7.78",
+  "VERSION_NUMBER": "0.7.78",
+  "BUILD_DATE": "2026-06-30T14:33:14.964Z"
 }

www/ndk-worker.js

@@ -6192,7 +6192,6 @@ async function handlePublish(requestId, event, port) {
         // temporary NDKRelaySet. NDK's fromRelayUrls uses pool.useTemporaryRelay
         // so hundreds of broadcast relays connect transiently without polluting
         // the read subscription pool.
-        let targetRelaySet = null;
         const activeBroadcastUrls = getActiveBroadcastUrls();
         const isBroadcast = wantsBroadcastRelays(event) && activeBroadcastUrls.length > 0;
         let outboxWriteUrls = [];
@@ -6201,31 +6200,24 @@ async function handlePublish(requestId, event, port) {
             outboxWriteUrls = Array.from(relayTypes.entries())
                 .filter(([, t]) => t === 'write' || t === 'both')
                 .map(([url]) => url);
-            const allUrls = [...new Set([...outboxWriteUrls, ...activeBroadcastUrls])];
-            if (NDKRelaySet?.fromRelayUrls) {
-                targetRelaySet = NDKRelaySet.fromRelayUrls(allUrls, ndk);
-                console.log(`[Worker] Broadcasting to ${allUrls.length} relays ` +
-                    `(${activeBroadcastUrls.length} broadcast + ${outboxWriteUrls.length} outbox, ` +
-                    `${skippedRelayUrls.size} skipped)`);
-            } else {
-                console.warn('[Worker] NDKRelaySet.fromRelayUrls unavailable, falling back to default outbox');
-            }
         }
+        const allBroadcastUrls = [...new Set([...outboxWriteUrls, ...activeBroadcastUrls])];
 
         // --- Live broadcast progress streaming ------------------------------------
         // NDK's NDKEvent emits 'relay:published' and 'relay:publish:failed' per
-        // relay as each completes (see ndk-core.bundle.js:8530 and :8552). We
-        // attach listeners BEFORE publish() and stream progress to all pages via
-        // broadcast() so footers can show "📡 42/150 relays · relay.example.com".
+        // relay as each completes. We attach listeners BEFORE publish() and stream
+        // progress to all pages via broadcast() so footers can show the count.
         const publishSessionId = `pub_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
         const publishedSoFar = new Set();
         const failedSoFar = new Set();       // explicit rejections (auth-required, blocked, etc.)
         const timedOutSoFar = new Set();     // timeouts — NOT skip-marked (could be slow connect)
-        const totalTarget = targetRelaySet && targetRelaySet.relays
-            ? targetRelaySet.relays.size
-            : (targetRelaySet ? (targetRelaySet.size || 0) : 0);
+        const totalTarget = isBroadcast ? allBroadcastUrls.length : 0;
 
         if (isBroadcast && totalTarget > 0) {
+            console.log(`[Worker] Broadcasting to ${allBroadcastUrls.length} relays ` +
+                `(${activeBroadcastUrls.length} broadcast + ${outboxWriteUrls.length} outbox, ` +
+                `${skippedRelayUrls.size} skipped)`);
+
             broadcast({
                 type: 'broadcastProgress',
                 sessionId: publishSessionId,
@@ -6249,7 +6241,7 @@ async function handlePublish(requestId, event, port) {
                     phase: 'progress',
                     total: totalTarget,
                     successful: publishedSoFar.size,
-                    failed: failedSoFar.size,
+                    failed: failedSoFar.size + timedOutSoFar.size,
                     latestRelay: url,
                 });
             });
@@ -6258,10 +6250,6 @@ async function handlePublish(requestId, event, port) {
                 const url = relay?.url || String(relay || '');
                 if (!url) return;
                 const errMsg = err?.message || String(err) || '';
-                // Distinguish timeouts from explicit rejections. Timeouts with
-                // hundreds of relays often just mean the relay was slow to connect,
-                // not that it's broken — so we track them separately and do NOT
-                // skip-mark them. Only explicit rejections go into failedSoFar.
                 const isTimeout = /timeout/i.test(errMsg);
                 if (isTimeout) {
                     timedOutSoFar.add(url);
@@ -6281,46 +6269,152 @@ async function handlePublish(requestId, event, port) {
             });
         }
 
-        // Publish to relays (with broadcast relay set if broadcasting, else default outbox).
-        // When broadcasting to many relays (potentially hundreds), use a much longer
-        // timeout than NDK's default 4400ms — temporary relays need time to establish
-        // WebSocket connections. Use 30s for broadcasts, default for normal publishes.
-        // requiredRelayCount=1 so NDK doesn't throw if only 1 of 600 relays succeeds.
-        const BROADCAST_TIMEOUT_MS = 30000;
-        const publishTimeoutMs = isBroadcast ? BROADCAST_TIMEOUT_MS : undefined;
-        const publishRequiredCount = isBroadcast ? 1 : undefined;
+        // --- Batched publishing for broadcasts ------------------------------------
+        // Browsers can only handle ~50-100 simultaneous WebSocket connections.
+        // Publishing to 600+ relays in one shot causes most connections to queue
+        // and time out before getting a slot. We split the relay URLs into batches
+        // of 25 and publish to each batch sequentially, keeping the number of
+        // simultaneous connections manageable.
+        const BATCH_SIZE = 25;
+        const PER_RELAY_TIMEOUT_MS = 10000; // 10s per relay within a batch
 
-        let relaySet;
-        try {
-            relaySet = await ndkEvent.publish(targetRelaySet, publishTimeoutMs, publishRequiredCount);
-        } catch (publishError) {
-            // NDK throws NDKPublishError if requiredRelayCount isn't met. With
-            // requiredRelayCount=1 this shouldn't happen for broadcasts, but handle
-            // it gracefully — the live progress listeners already captured whatever
-            // succeeded. Log and continue so we still report results to the page.
-            console.warn('[Worker] publish() threw (some relays may still have succeeded):', publishError?.message || publishError);
-            relaySet = null;
-        }
+        let relaySet = null;
+        if (isBroadcast && totalTarget > 0 && NDKRelaySet?.fromRelayUrls) {
+            // --- Phase 1: Publish to the user's standard outbox relays first ---
+            // This ensures the post lands on the user's primary relays quickly,
+            // and lets us send the response back to the page so the composer
+            // clears immediately. The broadcast relay batches run afterward.
+            if (outboxWriteUrls.length > 0) {
+                console.log(`[Worker] Phase 1: Publishing to ${outboxWriteUrls.length} outbox relays first`);
+                try {
+                    const outboxSet = NDKRelaySet.fromRelayUrls(outboxWriteUrls, ndk);
+                    await ndkEvent.publish(outboxSet, 10000, 1);
+                } catch (outboxErr) {
+                    console.warn('[Worker] Outbox publish error (expected if some relays fail):', outboxErr?.message || outboxErr);
+                }
+                console.log(`[Worker] Phase 1 complete: ${publishedSoFar.size} outbox relays succeeded`);
+            }
 
-        // Final broadcast progress message with complete results.
-        if (isBroadcast && totalTarget > 0) {
+            // --- Send response back to the page NOW so the composer clears ---
+            // The post is already on the user's outbox relays. The broadcast
+            // batches continue in the background.
+            const earlyRelayResults = {
+                successful: Array.from(publishedSoFar),
+                failed: [],
+            };
+            port.postMessage({
+                type: 'response',
+                requestId: requestId,
+                data: {
+                    success: true,
+                    relayResults: earlyRelayResults,
+                    totalRelays: earlyRelayResults.successful.length,
+                }
+            });
+            console.log('[Worker] Sent early publish response — composer can clear now');
+
+            // --- Phase 2: Broadcast to remaining relays in batches (background) ---
+            // Only publish to relays that aren't already in the outbox set
+            // (those were already published in Phase 1).
+            const broadcastOnlyUrls = activeBroadcastUrls.filter(u => !outboxWriteUrls.includes(u));
+            if (broadcastOnlyUrls.length > 0) {
+                const batches = [];
+                for (let i = 0; i < broadcastOnlyUrls.length; i += BATCH_SIZE) {
+                    batches.push(broadcastOnlyUrls.slice(i, i + BATCH_SIZE));
+                }
+                console.log(`[Worker] Phase 2: Broadcasting to ${broadcastOnlyUrls.length} relays in ${batches.length} batches of ${BATCH_SIZE}`);
+
+                for (let bi = 0; bi < batches.length; bi++) {
+                    const batch = batches[bi];
+                    const batchSet = NDKRelaySet.fromRelayUrls(batch, ndk);
+                    if (batchSet?.relays) {
+                        for (const relay of batchSet.relays) {
+                            try {
+                                if (relay.connectionTimeout !== undefined) {
+                                    relay.connectionTimeout = PER_RELAY_TIMEOUT_MS;
+                                }
+                            } catch (_) { /* relay may be read-only */ }
+                        }
+                    }
+                    try {
+                        await ndkEvent.publish(batchSet, PER_RELAY_TIMEOUT_MS, 1);
+                    } catch (batchErr) {
+                        // Expected — some relays in this batch may fail.
+                    }
+                    console.log(`[Worker] Batch ${bi + 1}/${batches.length} complete — ` +
+                        `${publishedSoFar.size}/${totalTarget} total succeeded, ` +
+                        `${failedSoFar.size + timedOutSoFar.size} failed/timeout so far`);
+                }
+            }
+
+            // Final broadcast progress message with complete results.
             const totalFailed = failedSoFar.size + timedOutSoFar.size;
             broadcast({
                 type: 'broadcastProgress',
                 sessionId: publishSessionId,
+                eventId: ndkEvent.id || null,
+                eventKind: event.kind,
                 phase: 'done',
                 total: totalTarget,
                 successful: publishedSoFar.size,
                 failed: totalFailed,
                 latestRelay: null,
+                relayUrls: Array.from(publishedSoFar),
             });
-            // Clean up listeners so the NDKEvent can be garbage-collected.
             try {
                 ndkEvent.removeAllListeners('relay:published');
                 ndkEvent.removeAllListeners('relay:publish:failed');
-            } catch (_cleanupErr) {
-                // removeAllListeners may not exist on all EventEmitter impls; ignore.
+            } catch (_cleanupErr) { /* ignore */ }
+
+            // --- Failure marking (background, after all batches) ---
+            const newSkips = Array.from(failedSoFar)
+                .filter(url => broadcastRelayUrls.has(url) && !skippedRelayUrls.has(url));
+            if (newSkips.length > 0) {
+                for (const url of newSkips) {
+                    skippedRelayUrls.set(url, {
+                        reason: 'publish-rejected',
+                        timestamp: Math.floor(Date.now() / 1000),
+                    });
+                }
+                scheduleSkipMarkSave();
+                console.log(`[Worker] Skip-marked ${newSkips.length} rejected broadcast relays ` +
+                    `(${timedOutSoFar.size} timeouts excluded)`);
             }
+
+            // Return early — we already sent the response above. The rest of
+            // handlePublish (relayResults, kind 10002 sync, etc.) is skipped for
+            // broadcasts since we've handled everything here.
+            return;
+        } else {
+            // Normal (non-broadcast) publish — await the full result as before.
+            try {
+                relaySet = await ndkEvent.publish(null, undefined, undefined);
+            } catch (publishError) {
+                console.warn('[Worker] publish() threw:', publishError?.message || publishError);
+                relaySet = null;
+            }
+        }
+
+        // For non-broadcast publishes, emit a simplified broadcastProgress 'done'
+        // event so post cards can still show the relay count (e.g., "3r - 1h").
+        // We use the relaySet result to count successful relays.
+        if (!isBroadcast) {
+            const successCount = relaySet ? relaySet.size : 0;
+            const successUrls = relaySet
+                ? Array.from(relaySet).map(r => r?.url || '').filter(Boolean)
+                : [];
+            broadcast({
+                type: 'broadcastProgress',
+                sessionId: publishSessionId,
+                eventId: ndkEvent.id || null,
+                eventKind: event.kind,
+                phase: 'done',
+                total: successCount,
+                successful: successCount,
+                failed: 0,
+                latestRelay: null,
+                relayUrls: successUrls,
+            });
         }
 
         // Get detailed relay results

www/post.html

@@ -833,12 +833,8 @@ import { initPostCards } from './js/post-interactions2.mjs';
               : '';
             spanBroadcastStatus.textContent = `📡 ${d.successful}/${d.total} relays · ${shortRelay}`;
           } else if (d.phase === 'done') {
+            // Keep the result displayed until the next publish overwrites it.
             spanBroadcastStatus.textContent = `✅ ${d.successful}/${d.total} relays` + (d.failed > 0 ? ` (${d.failed} failed)` : '');
-            setTimeout(() => {
-              if (spanBroadcastStatus.textContent.startsWith('✅')) {
-                spanBroadcastStatus.textContent = '';
-              }
-            }, 10000);
           }
         });