nip44: support encryption of payloads larger than 65535 bytes (#527)

* nip44: support encryption of payloads larger than 65535 bytes

Implement the extended prefix format from nostr-protocol/nips#1907
with bug fixes:

- When plaintext length >= 65536, pad with a 6-byte prefix (2 zero
  sentinel bytes + 4-byte u32 BE length) instead of the 2-byte u16
- Detect the sentinel in unpad() and use dynamic prefix_len (2 or 6)
  for both data extraction and padding validation
- Remove upper-bound size limits in decodePayload() (was rejecting
  payloads > 87472 chars / 65603 bytes)
- Raise maxPlaintextSize from 0xffff to 0xffffffff (2^32-1)
- Add writeU32BE() helper for encoding large lengths
- Fix unpad() to use padded.byteOffset for correct DataView on
  subarrays (was using padded.buffer directly which could be wrong
  if padded is a subarray of a larger buffer)

Tests: add boundary tests at 65535/65536/65537 bytes and full
encrypt/decrypt round-trips at 65536 and 100000 bytes.
All 10 tests pass (5 existing + 5 new).

* nip44: fix calcPaddedLen overflow, enforce canonical prefix, add thorough tests

Fix three bugs found during security audit:

- calcPaddedLen: use 2** instead of 1<< to avoid signed 32-bit
  overflow in JS for plaintext lengths above 2^30
- unpad: reject extended 6-byte prefix when decoded length is below
  65536 (enforce canonical encoding, prevent ambiguous padding)
- decodePayload: document that callers should validate payload size
  before calling to prevent DoS from oversized inputs
- Fix stale size-range comments to match corrected spec arithmetic

Add 10 new tests covering:
- Non-canonical extended prefix rejection (len=1, 1000, 65535)
- Truncated extended prefix buffer
- calcPaddedLen at 2^30+1, 2^31, and 2^32-1
- Multi-byte UTF-8 at the 65536 byte boundary (pad/unpad + e2e)
- Spec test vectors with SHA-256 checksums for 65535/65536/65537

nip44.test.ts

@@ -3,6 +3,7 @@ import { v2 } from './nip44.js'
 import { bytesToHex, hexToBytes } from '@noble/hashes/utils.js'
 import { default as vec } from './nip44.vectors.json' with { type: 'json' }
 import { schnorr } from '@noble/curves/secp256k1.js'
+
 const v2vec = vec.v2
 
 test('get_conversation_key', () => {
@@ -44,3 +45,231 @@ test('get_conversation_key', async () => {
     )
   }
 })
+
+// Extended prefix (big payload) tests
+test('pad/unpad boundary: 65535 bytes uses 2-byte u16 prefix', () => {
+  const plaintext = 'a'.repeat(65535)
+  const padded = v2.utils.pad(plaintext)
+  // First 2 bytes should be 0xff 0xff (65535 as u16 BE)
+  expect(padded[0]).toEqual(0xff)
+  expect(padded[1]).toEqual(0xff)
+  const unpadded = v2.utils.unpad(padded)
+  expect(unpadded).toEqual(plaintext)
+})
+
+test('pad/unpad boundary: 65536 bytes uses 6-byte extended prefix', () => {
+  const plaintext = 'a'.repeat(65536)
+  const padded = v2.utils.pad(plaintext)
+  // First 2 bytes should be 0x00 0x00 (sentinel)
+  expect(padded[0]).toEqual(0x00)
+  expect(padded[1]).toEqual(0x00)
+  // Next 4 bytes should be 0x00 0x01 0x00 0x00 (65536 as u32 BE)
+  expect(padded[2]).toEqual(0x00)
+  expect(padded[3]).toEqual(0x01)
+  expect(padded[4]).toEqual(0x00)
+  expect(padded[5]).toEqual(0x00)
+  const unpadded = v2.utils.unpad(padded)
+  expect(unpadded).toEqual(plaintext)
+})
+
+test('pad/unpad boundary: 65537 bytes uses 6-byte extended prefix', () => {
+  const plaintext = 'a'.repeat(65537)
+  const padded = v2.utils.pad(plaintext)
+  // First 2 bytes should be sentinel
+  expect(padded[0]).toEqual(0x00)
+  expect(padded[1]).toEqual(0x00)
+  // Next 4 bytes should be 0x00 0x01 0x00 0x01 (65537 as u32 BE)
+  expect(padded[2]).toEqual(0x00)
+  expect(padded[3]).toEqual(0x01)
+  expect(padded[4]).toEqual(0x00)
+  expect(padded[5]).toEqual(0x01)
+  const unpadded = v2.utils.unpad(padded)
+  expect(unpadded).toEqual(plaintext)
+})
+
+test('encrypt/decrypt round-trip with big payload (65536 bytes)', () => {
+  const plaintext = 'x'.repeat(65536)
+  const sec1 = hexToBytes('0000000000000000000000000000000000000000000000000000000000000001')
+  const sec2 = hexToBytes('0000000000000000000000000000000000000000000000000000000000000002')
+  const pub2 = bytesToHex(schnorr.getPublicKey(sec2))
+  const conversationKey = v2.utils.getConversationKey(sec1, pub2)
+  const encrypted = v2.encrypt(plaintext, conversationKey)
+  const decrypted = v2.decrypt(encrypted, conversationKey)
+  expect(decrypted).toEqual(plaintext)
+})
+
+test('encrypt/decrypt round-trip with 100000 byte payload', () => {
+  const plaintext = 'z'.repeat(100000)
+  const sec1 = hexToBytes('0000000000000000000000000000000000000000000000000000000000000001')
+  const sec2 = hexToBytes('0000000000000000000000000000000000000000000000000000000000000002')
+  const pub2 = bytesToHex(schnorr.getPublicKey(sec2))
+  const conversationKey = v2.utils.getConversationKey(sec1, pub2)
+  const encrypted = v2.encrypt(plaintext, conversationKey)
+  const decrypted = v2.decrypt(encrypted, conversationKey)
+  expect(decrypted).toEqual(plaintext)
+})
+
+// Canonicality: reject non-canonical extended prefix for small lengths
+test('unpad rejects non-canonical extended prefix for length=1', () => {
+  const unpaddedLen = 1
+  const calcPaddedLen = v2.utils.calcPaddedLen(unpaddedLen) // 32
+  const buf = new Uint8Array(6 + calcPaddedLen) // 6-byte prefix + 32 bytes padded
+  buf[0] = 0x00
+  buf[1] = 0x00 // sentinel
+  buf[2] = 0x00
+  buf[3] = 0x00
+  buf[4] = 0x00
+  buf[5] = 0x01 // u32 BE = 1
+  buf[6] = 0x61 // 'a'
+  expect(() => v2.utils.unpad(buf)).toThrow(/invalid padding/)
+})
+
+test('unpad rejects non-canonical extended prefix for length=1000', () => {
+  const unpaddedLen = 1000
+  const calcPaddedLen = v2.utils.calcPaddedLen(unpaddedLen) // 1024
+  const buf = new Uint8Array(6 + calcPaddedLen)
+  buf[0] = 0x00
+  buf[1] = 0x00 // sentinel
+  buf[2] = 0x00
+  buf[3] = 0x00
+  buf[4] = 0x03
+  buf[5] = 0xe8 // u32 BE = 1000
+  for (let i = 0; i < unpaddedLen; i++) buf[6 + i] = 0x61 // 'a'
+  expect(() => v2.utils.unpad(buf)).toThrow(/invalid padding/)
+})
+
+test('unpad rejects non-canonical extended prefix for length=65535', () => {
+  const unpaddedLen = 65535
+  const calcPaddedLen = v2.utils.calcPaddedLen(unpaddedLen) // 65536
+  const buf = new Uint8Array(6 + calcPaddedLen)
+  buf[0] = 0x00
+  buf[1] = 0x00 // sentinel
+  buf[2] = 0x00
+  buf[3] = 0x00
+  buf[4] = 0xff
+  buf[5] = 0xff // u32 BE = 65535
+  for (let i = 0; i < unpaddedLen; i++) buf[6 + i] = 0x61
+  expect(() => v2.utils.unpad(buf)).toThrow(/invalid padding/)
+})
+
+// Malformed extended prefix: buffer too short for the 6-byte header
+test('unpad rejects truncated extended prefix (buffer shorter than 6 bytes)', () => {
+  const buf = new Uint8Array([0x00, 0x00, 0x00, 0x01])
+  expect(() => v2.utils.unpad(buf)).toThrow()
+})
+
+// calcPaddedLen must not overflow for large values (regression: 1 << 31 is negative in JS)
+test('calcPaddedLen handles values above 2^30 correctly', () => {
+  const len = 2 ** 30 + 1 // 1073741825
+  const padded = v2.utils.calcPaddedLen(len)
+  expect(padded).toBeGreaterThanOrEqual(len)
+  // chunk = 2^31 / 8 = 268435456
+  expect(padded % 268435456).toEqual(0)
+})
+
+test('calcPaddedLen handles 2^31 correctly', () => {
+  const len = 2 ** 31 // 2147483648
+  const padded = v2.utils.calcPaddedLen(len)
+  expect(padded).toBeGreaterThanOrEqual(len)
+  // chunk = 2^32 / 8 = 536870912
+  expect(padded % 536870912).toEqual(0)
+})
+
+test('calcPaddedLen handles max plaintext size (2^32 - 1)', () => {
+  const len = 0xffffffff // 4294967295
+  const padded = v2.utils.calcPaddedLen(len)
+  expect(padded).toBeGreaterThanOrEqual(len)
+  // Must be exactly 2^32 = 4294967296
+  expect(padded).toEqual(4294967296)
+})
+
+// Multi-byte UTF-8 near the 65536 boundary: byte length != char length
+test('pad/unpad with multi-byte UTF-8 near 65536 byte boundary', () => {
+  // U+00E9 (é) is 2 bytes in UTF-8. Use 32768 of them = 65536 bytes but 32768 chars
+  const plaintext = '\u00e9'.repeat(32768)
+  const encoded = new TextEncoder().encode(plaintext)
+  expect(encoded.length).toEqual(65536) // byte length triggers extended prefix
+  expect(plaintext.length).toEqual(32768) // char length is much smaller
+
+  const padded = v2.utils.pad(plaintext)
+  // Should use extended prefix since byte length is 65536
+  expect(padded[0]).toEqual(0x00)
+  expect(padded[1]).toEqual(0x00)
+  const unpadded = v2.utils.unpad(padded)
+  expect(unpadded).toEqual(plaintext)
+})
+
+test('encrypt/decrypt with multi-byte UTF-8 at 65536 bytes', () => {
+  const plaintext = '\u00e9'.repeat(32768) // 65536 bytes
+  const sec1 = hexToBytes('0000000000000000000000000000000000000000000000000000000000000001')
+  const sec2 = hexToBytes('0000000000000000000000000000000000000000000000000000000000000002')
+  const pub2 = bytesToHex(schnorr.getPublicKey(sec2))
+  const conversationKey = v2.utils.getConversationKey(sec1, pub2)
+  const encrypted = v2.encrypt(plaintext, conversationKey)
+  const decrypted = v2.decrypt(encrypted, conversationKey)
+  expect(decrypted).toEqual(plaintext)
+})
+
+// Spec test vectors: SHA-256 checksums for boundary payloads
+test('spec test vectors: SHA-256 of payload at 65535/65536/65537', async () => {
+  const convKey = hexToBytes('c41c775356fd92eadc63ff5a0dc1da211b268cbea22316767095b2871ea1412d')
+  const nonce = hexToBytes('0000000000000000000000000000000000000000000000000000000000000001')
+
+  async function sha256hex(data: Uint8Array): Promise<string> {
+    const hash = await crypto.subtle.digest('SHA-256', data)
+    return bytesToHex(new Uint8Array(hash))
+  }
+
+  const vectors = [
+    {
+      len: 65535,
+      prefix: 'u16',
+      padded_len: 65536,
+      plaintext_sha256: '6e1bebca6a8229364a162a72ef064826c4cd7457bf54f190ef782bd9deff3e42',
+      payload_sha256: '6d8c2810d1e870fbaa1f0a0937126cca837a15f9260e27060c331d70a3c0bc84',
+    },
+    {
+      len: 65536,
+      prefix: 'extended',
+      padded_len: 65536,
+      plaintext_sha256: 'bf718b6f653bebc184e1479f1935b8da974d701b893afcf49e701f3e2f9f9c5a',
+      payload_sha256: 'b7b4edb36ba92e267d322d56d9aebc22e7fa96ff52e3c12adc07f07a43cbc616',
+    },
+    {
+      len: 65537,
+      prefix: 'extended',
+      padded_len: 81920,
+      plaintext_sha256: '008ffc88d3c96a9f307524eb361e47c5222a887fc45fa0c1fb8d429c5c23b430',
+      payload_sha256: 'eeb7c7c5373894ea2c1547cfd3ccb15d5a0b2d619da852e5c79df792dcc9e435',
+    },
+  ]
+
+  for (const vec of vectors) {
+    const plaintext = 'a'.repeat(vec.len)
+    const ptBytes = new TextEncoder().encode(plaintext)
+
+    // Verify plaintext SHA-256
+    expect(await sha256hex(ptBytes)).toEqual(vec.plaintext_sha256)
+
+    // Verify padded length
+    expect(v2.utils.calcPaddedLen(vec.len)).toEqual(vec.padded_len)
+
+    // Verify prefix type
+    const padded = v2.utils.pad(plaintext)
+    if (vec.prefix === 'u16') {
+      expect(padded[0]).not.toEqual(0)
+    } else {
+      expect(padded[0]).toEqual(0)
+      expect(padded[1]).toEqual(0)
+    }
+
+    // Encrypt and verify payload SHA-256
+    const payload = v2.encrypt(plaintext, convKey, nonce)
+    const payloadBytes = new TextEncoder().encode(payload)
+    expect(await sha256hex(payloadBytes)).toEqual(vec.payload_sha256)
+
+    // Verify round-trip decrypt
+    const decrypted = v2.decrypt(payload, convKey)
+    expect(decrypted).toEqual(plaintext)
+  }
+})

nip44.ts

@@ -10,7 +10,8 @@ import { base64 } from '@scure/base'
 import { utf8Decoder, utf8Encoder } from './utils.ts'
 
 const minPlaintextSize = 0x0001 // 1b msg => padded to 32b
-const maxPlaintextSize = 0xffff // 65535 (64kb-1) => padded to 64kb
+const maxPlaintextSize = 0xffffffff // 4294967295 (2^32-1)
+const extendedPrefixThreshold = 0x10000 // 65536: lengths below use 2-byte u16 prefix, at or above use 6-byte prefix
 
 export function getConversationKey(privkeyA: Uint8Array, pubkeyB: string): Uint8Array {
   const sharedX = secp256k1.getSharedSecret(privkeyA, hexToBytes('02' + pubkeyB)).subarray(1, 33)
@@ -32,35 +33,60 @@ function getMessageKeys(
 function calcPaddedLen(len: number): number {
   if (!Number.isSafeInteger(len) || len < 1) throw new Error('expected positive integer')
   if (len <= 32) return 32
-  const nextPower = 1 << (Math.floor(Math.log2(len - 1)) + 1)
+  const nextPower = 2 ** (Math.floor(Math.log2(len - 1)) + 1)
   const chunk = nextPower <= 256 ? 32 : nextPower / 8
   return chunk * (Math.floor((len - 1) / chunk) + 1)
 }
 
 function writeU16BE(num: number): Uint8Array {
-  if (!Number.isSafeInteger(num) || num < minPlaintextSize || num > maxPlaintextSize)
+  if (!Number.isSafeInteger(num) || num < minPlaintextSize || num > 0xffff)
     throw new Error('invalid plaintext size: must be between 1 and 65535 bytes')
   const arr = new Uint8Array(2)
   new DataView(arr.buffer).setUint16(0, num, false)
   return arr
 }
 
+function writeU32BE(num: number): Uint8Array {
+  if (!Number.isSafeInteger(num) || num < extendedPrefixThreshold || num > maxPlaintextSize)
+    throw new Error('invalid plaintext size: must be between 65536 and 4294967295 bytes')
+  const arr = new Uint8Array(4)
+  new DataView(arr.buffer).setUint32(0, num, false)
+  return arr
+}
+
 function pad(plaintext: string): Uint8Array {
   const unpadded = utf8Encoder.encode(plaintext)
   const unpaddedLen = unpadded.length
-  const prefix = writeU16BE(unpaddedLen)
+  if (unpaddedLen < minPlaintextSize || unpaddedLen > maxPlaintextSize)
+    throw new Error('invalid plaintext size: must be between 1 and 4294967295 bytes')
+  const prefix =
+    unpaddedLen >= extendedPrefixThreshold
+      ? concatBytes(new Uint8Array([0, 0]), writeU32BE(unpaddedLen)) // 6 bytes
+      : writeU16BE(unpaddedLen) // 2 bytes
   const suffix = new Uint8Array(calcPaddedLen(unpaddedLen) - unpaddedLen)
   return concatBytes(prefix, unpadded, suffix)
 }
 
 function unpad(padded: Uint8Array): string {
-  const unpaddedLen = new DataView(padded.buffer).getUint16(0)
+  const dv = new DataView(padded.buffer, padded.byteOffset, padded.byteLength)
-  const unpadded = padded.subarray(2, 2 + unpaddedLen)
+  const firstTwo = dv.getUint16(0)
+  let unpaddedLen: number
+  let prefixLen: number
+  if (firstTwo === 0) {
+    // Extended format: 2 zero bytes + 4-byte u32 length
+    unpaddedLen = dv.getUint32(2)
+    if (unpaddedLen < extendedPrefixThreshold) throw new Error('invalid padding')
+    prefixLen = 6
+  } else {
+    unpaddedLen = firstTwo
+    prefixLen = 2
+  }
+  const unpadded = padded.subarray(prefixLen, prefixLen + unpaddedLen)
   if (
     unpaddedLen < minPlaintextSize ||
     unpaddedLen > maxPlaintextSize ||
     unpadded.length !== unpaddedLen ||
-    padded.length !== 2 + calcPaddedLen(unpaddedLen)
+    padded.length !== prefixLen + calcPaddedLen(unpaddedLen)
   )
     throw new Error('invalid padding')
   return utf8Decoder.decode(unpadded)
@@ -72,16 +98,17 @@ function hmacAad(key: Uint8Array, message: Uint8Array, aad: Uint8Array): Uint8Ar
   return hmac(sha256, key, combined)
 }
 
-// metadata: always 65b (version: 1b, nonce: 32b, max: 32b)
+// metadata: always 65b (version: 1b, nonce: 32b, mac: 32b)
-// plaintext: 1b to 0xffff
+// plaintext: 1b to 0xffffffff
-// padded plaintext: 32b to 0xffff
+// padded plaintext (small, <65536): 32b to 0x10000, with 2b prefix -> 34b to 0x10000+2
-// ciphertext: 32b+2 to 0xffff+2
+// padded plaintext (large, >=65536): 0x10000 to 0x100000000, with 6b prefix -> 0x10006 to 0x100000000+6
-// raw payload: 99 (65+32+2) to 65603 (65+0xffff+2)
+// ciphertext: same as padded plaintext (chacha20 doesn't change length)
-// compressed payload (base64): 132b to 87472b
+// raw payload (small): 99 (65+34) to 65603 (65+0x10000+2)
+// raw payload (large): 65607 (65+0x10006) to 4294967367 (65+0x100000000+6)
 function decodePayload(payload: string): { nonce: Uint8Array; ciphertext: Uint8Array; mac: Uint8Array } {
   if (typeof payload !== 'string') throw new Error('payload must be a valid string')
   const plen = payload.length
-  if (plen < 132 || plen > 87472) throw new Error('invalid payload length: ' + plen)
+  if (plen < 132) throw new Error('invalid payload length: ' + plen)
   if (payload[0] === '#') throw new Error('unknown encryption version')
   let data: Uint8Array
   try {
@@ -90,7 +117,7 @@ function decodePayload(payload: string): { nonce: Uint8Array; ciphertext: Uint8A
     throw new Error('invalid base64: ' + (error as any).message)
   }
   const dlen = data.length
-  if (dlen < 99 || dlen > 65603) throw new Error('invalid data length: ' + dlen)
+  if (dlen < 99) throw new Error('invalid data length: ' + dlen)
   const vers = data[0]
   if (vers !== 2) throw new Error('unknown encryption version ' + vers)
   return {
@@ -108,6 +135,7 @@ export function encrypt(plaintext: string, conversationKey: Uint8Array, nonce: U
   return base64.encode(concatBytes(new Uint8Array([2]), nonce, ciphertext, mac))
 }
 
+/** Callers should validate payload size before calling to prevent DoS from oversized inputs. */
 export function decrypt(payload: string, conversationKey: Uint8Array): string {
   const { nonce, ciphertext, mac } = decodePayload(payload)
   const { chacha_key, chacha_nonce, hmac_key } = getMessageKeys(conversationKey, nonce)
@@ -121,6 +149,8 @@ export const v2 = {
   utils: {
     getConversationKey,
     calcPaddedLen,
+    pad,
+    unpad,
   },
   encrypt,
   decrypt,