nip45 helpers and pool.countMany() with HLL.

abstract-pool.ts

@@ -11,6 +11,7 @@ import { normalizeURL } from './utils.ts'
 import type { Event, EventTemplate, Nostr, VerifiedEvent } from './core.ts'
 import { type Filter } from './filter.ts'
 import { alwaysTrue } from './helpers.ts'
+import { getCountManyFilter, hllDecode, hllEncode, mergeHll, type CountManyDirective } from './nip45.ts'
 import { Relay } from './relay.ts'
 
 export type SubCloser = { close: (reason?: string) => void }
@@ -318,6 +319,63 @@ export class AbstractSimplePool {
     return events[0] || null
   }
 
+  async countMany(
+    relays: string[],
+    target: string,
+    directive: CountManyDirective,
+    params?: { id?: string | null; maxWait?: number; abort?: AbortSignal },
+  ): Promise<{ count: number; hll?: string }> {
+    const filter = getCountManyFilter(target, directive)
+    const urls: string[] = []
+
+    for (let i = 0; i < relays.length; i++) {
+      const url = normalizeURL(relays[i])
+      if (urls.indexOf(url) === -1) urls.push(url)
+    }
+
+    const responses = await Promise.all(
+      urls.map(async url => {
+        if (this.allowConnectingToRelay?.(url, ['read', [filter]]) === false) return null
+
+        let relay: AbstractRelay
+        try {
+          relay = await this.ensureRelay(url, {
+            connectionTimeout:
+              this.maxWaitForConnection < (params?.maxWait || 0)
+                ? Math.max(params!.maxWait! * 0.8, params!.maxWait! - 1000)
+                : this.maxWaitForConnection,
+            abort: params?.abort,
+          })
+        } catch (err) {
+          this.onRelayConnectionFailure?.(url)
+          return null
+        }
+
+        this.onRelayConnectionSuccess?.(url)
+
+        return relay.countWithHLL([filter], { id: params?.id }).catch(() => null)
+      }),
+    )
+
+    let count = 0
+    let hll: Uint8Array | undefined
+
+    for (const response of responses) {
+      if (!response) continue
+
+      if (response.count > count) count = response.count
+
+      if (!response.hll || response.hll.length !== 512) continue
+
+      const registers = hllDecode(response.hll)
+      if (!registers) continue
+
+      hll = mergeHll(hll || new Uint8Array(0), registers)
+    }
+
+    return hll ? { count, hll: hllEncode(hll) } : { count }
+  }
+
   publish(
     relays: string[],
     event: Event,

abstract-relay.ts

@@ -496,7 +496,14 @@ export class AbstractRelay {
         case 'CLOSED': {
           const id: string = data[1]
           const so = this.openSubs.get(id)
-          if (!so) return
+          if (!so) {
+            const cr = this.openCountRequests.get(id) as CountResolver
+            if (cr) {
+              cr.reject(new Error(data[2] as string))
+              this.openCountRequests.delete(id)
+            }
+            return
+          }
           so.closed = true
           so.close(data[2] as string)
           return

nip45.test.ts

@@ -0,0 +1,58 @@
+import { expect, test } from 'bun:test'
+
+import { computeOffset, estimateCount, feedPubkey, hllDecode, hllEncode, mergeHll, newHll } from './nip45.ts'
+
+const pubkeys = [
+  '0000000000000000000000000000000000000000000000000000000000000000',
+  'ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff',
+  '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef',
+  'abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789',
+  '0000000000000000010000000000000000000000000000000000000000000000',
+  '0000000000000000028000000000000000000000000000000000000000000000',
+  '0000000000000000024000000000000000000000000000000000000000000000',
+]
+
+const goRegisters =
+  '39390200000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001'
+
+test('matches nostrlib hyperloglog registers and count', () => {
+  let hll = newHll()
+  for (const pubkey of pubkeys) {
+    hll = feedPubkey(hll, pubkey, 8)
+  }
+
+  expect(hllEncode(hll)).toBe(goRegisters)
+  expect(estimateCount(hll)).toBe(5)
+})
+
+test('matches nostrlib hyperloglog merge', () => {
+  let full = newHll()
+  let left = newHll()
+  let right = newHll()
+
+  pubkeys.forEach((pubkey, i) => {
+    full = feedPubkey(full, pubkey, 8)
+    if (i % 2 === 0) left = feedPubkey(left, pubkey, 8)
+    else right = feedPubkey(right, pubkey, 8)
+  })
+
+  left = mergeHll(left, right)
+
+  expect(hllEncode(left)).toBe(hllEncode(full))
+  expect(hllEncode(left)).toBe(goRegisters)
+  expect(estimateCount(left)).toBe(5)
+})
+
+test('matches nostrlib MergeRegisters behavior', () => {
+  const registers = hllDecode(goRegisters)
+  expect(registers).toBeDefined()
+
+  const hll = mergeHll(newHll(), registers!)
+  expect(hllEncode(hll)).toBe(goRegisters)
+  expect(estimateCount(hll)).toBe(5)
+})
+
+test('computes NIP-45 offset', () => {
+  expect(computeOffset('00000000000000000000000000000000f0000000000000000000000000000000')).toBe(23)
+  expect(computeOffset('30023:00000000000000000000000000000000a0000000000000000000000000000000:test')).toBe(18)
+})

nip45.ts

@@ -0,0 +1,167 @@
+import { sha256 } from '@noble/hashes/sha2.js'
+import { bytesToHex } from '@noble/hashes/utils.js'
+
+import type { Event } from './core.ts'
+import type { Filter } from './filter.ts'
+
+const M = 256
+const HLL_HEX_LENGTH = M * 2
+const utf8Encoder = new TextEncoder()
+
+export type CountManyDirective = 'reactions' | 'reposts' | 'quotes' | 'replies' | 'comments' | 'followers'
+
+export function getCountManyFilter(target: string, directive: CountManyDirective): Filter {
+  switch (directive) {
+    case 'reactions':
+      return { '#e': [target], kinds: [7] }
+    case 'reposts':
+      return { '#e': [target], kinds: [6] }
+    case 'quotes':
+      return { '#q': [target], kinds: [1, 1111] }
+    case 'replies':
+      return { '#e': [target], kinds: [1] }
+    case 'comments':
+      return { '#E': [target], kinds: [1111] }
+    case 'followers':
+      return { '#p': [target], kinds: [3] }
+  }
+}
+
+export function newHll(): Uint8Array {
+  return new Uint8Array(M)
+}
+
+export function hllDecode(hex: string): Uint8Array | undefined {
+  if (hex.length !== HLL_HEX_LENGTH || !/^[0-9a-fA-F]+$/.test(hex)) return undefined
+
+  const registers = new Uint8Array(M)
+  for (let i = 0; i < M; i++) {
+    registers[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16)
+  }
+  return registers
+}
+
+export function hllEncode(registers: Uint8Array): string {
+  if (registers.length !== M) throw new Error(`invalid number of registers ${registers.length}`)
+
+  let hex = ''
+  for (let i = 0; i < M; i++) {
+    hex += registers[i].toString(16).padStart(2, '0')
+  }
+  return hex
+}
+
+export function computeOffset(filterFirstTagValue: string): number {
+  let hex = filterFirstTagValue
+
+  if (!isHex64(hex)) {
+    const parts = hex.split(':')
+    if (parts.length === 3 && isHex64(parts[1])) {
+      hex = parts[1]
+    } else {
+      hex = bytesToHex(sha256(utf8Encoder.encode(filterFirstTagValue)))
+    }
+  }
+
+  return parseInt(hex[32], 16) + 8
+}
+
+export function getFilterFirstTagValue(filter: Filter): string | undefined {
+  for (const key in filter) {
+    if (key[0] !== '#') continue
+
+    const values = filter[key as `#${string}`]
+    if (Array.isArray(values) && typeof values[0] === 'string') return values[0]
+  }
+
+  return undefined
+}
+
+export function feedPubkey(hll: Uint8Array, pubkey: string, offset: number): Uint8Array {
+  if (offset < 0 || offset > 24) throw new Error(`invalid offset ${offset}`)
+  if (!isHex64(pubkey)) throw new Error('pubkey must be 32-byte hex')
+
+  if (hll.length === 0) hll = newHll()
+  if (hll.length !== M) throw new Error(`invalid number of registers ${hll.length}`)
+
+  const ri = parseInt(pubkey.slice(offset * 2, offset * 2 + 2), 16)
+  const value = countLeadingZeroBitsAfterOffset(pubkey, offset) + 1
+  if (value > hll[ri]) hll[ri] = value
+
+  return hll
+}
+
+export function feedEvent(hll: Uint8Array, event: Event, offset: number): Uint8Array {
+  return feedPubkey(hll, event.pubkey, offset)
+}
+
+export function mergeHll(target: Uint8Array, source: Uint8Array): Uint8Array {
+  if (target.length === 0) target = newHll()
+  if (target.length !== M) throw new Error(`invalid number of registers ${target.length}`)
+  if (source.length !== M) throw new Error(`invalid number of registers ${source.length}`)
+
+  for (let i = 0; i < M; i++) {
+    if (source[i] > target[i]) target[i] = source[i]
+  }
+
+  return target
+}
+
+export function estimateCount(hll: Uint8Array): number {
+  if (hll.length === 0) return 0
+  if (hll.length !== M) throw new Error(`invalid number of registers ${hll.length}`)
+
+  const v = countZeros(hll)
+  if (v !== 0) {
+    const lc = linearCounting(M, v)
+    if (lc <= 220) return Math.floor(lc)
+  }
+
+  const estimate = calculateEstimate(hll)
+  if (estimate <= M * 3 && v !== 0) return Math.floor(linearCounting(M, v))
+
+  return Math.floor(estimate)
+}
+
+function isHex64(value: string): boolean {
+  return /^[0-9a-fA-F]{64}$/.test(value)
+}
+
+function countLeadingZeroBitsAfterOffset(pubkey: string, offset: number): number {
+  let zeroBits = 0
+  for (let i = offset + 1; i < offset + 8; i++) {
+    const byte = parseInt(pubkey.slice(i * 2, i * 2 + 2), 16)
+    if (byte === 0) {
+      zeroBits += 8
+      continue
+    }
+
+    let mask = 0x80
+    while ((byte & mask) === 0) {
+      zeroBits++
+      mask >>= 1
+    }
+    break
+  }
+  return zeroBits
+}
+
+function countZeros(registers: Uint8Array): number {
+  let count = 0
+  for (let i = 0; i < M; i++) {
+    if (registers[i] === 0) count++
+  }
+  return count
+}
+
+function linearCounting(m: number, v: number): number {
+  return m * Math.log(m / v)
+}
+
+function calculateEstimate(registers: Uint8Array): number {
+  let sum = 0
+  for (let i = 0; i < M; i++) {
+    sum += 1 / 2 ** registers[i]
+  }
+  return (0.7182725932495458 * M * M) / sum
+}

package.json

@@ -198,6 +198,12 @@
       "require": "./lib/cjs/nip44.js",
       "types": "./lib/types/nip44.d.ts"
     },
+    "./nip45": {
+      "source": "./nip45.ts",
+      "import": "./lib/esm/nip45.js",
+      "require": "./lib/cjs/nip45.js",
+      "types": "./lib/types/nip45.d.ts"
+    },
     "./nip46": {
       "source": "./nip46.ts",
       "import": "./lib/esm/nip46.js",