only transform filters with enableReconnect() when we're actually reconnecting.

this was broken during the negentropy stuff.

fixes https://github.com/nbd-wtf/nostr-tools/issues/517

.eslintrc.json

@@ -138,6 +138,7 @@
     "valid-typeof": 2,
     "wrap-iife": [2, "any"],
     "yield-star-spacing": [2, "both"],
-    "yoda": [0]
+    "yoda": [0],
+    "no-labels": [0]
   }
 }

abstract-pool.ts

@@ -14,14 +14,17 @@ import { alwaysTrue } from './helpers.ts'
 
 export type SubCloser = { close: (reason?: string) => void }
 
-export type AbstractPoolConstructorOptions = AbstractRelayConstructorOptions & {}
+export type AbstractPoolConstructorOptions = AbstractRelayConstructorOptions & {
+  // automaticallyAuth takes a relay URL and should return null
+  // in case that relay shouldn't be authenticated against
+  // or a function to sign the AUTH event template otherwise (that function may still throw in case of failure)
+  automaticallyAuth?: (relayURL: string) => null | ((event: EventTemplate) => Promise<VerifiedEvent>)
+}
 
 export type SubscribeManyParams = Omit<SubscriptionParams, 'onclose'> & {
   maxWait?: number
   onclose?: (reasons: string[]) => void
   onauth?: (event: EventTemplate) => Promise<VerifiedEvent>
-  // Deprecated: use onauth instead
-  doauth?: (event: EventTemplate) => Promise<VerifiedEvent>
   id?: string
   label?: string
 }
@@ -34,6 +37,7 @@ export class AbstractSimplePool {
   public verifyEvent: Nostr['verifyEvent']
   public enablePing: boolean | undefined
   public enableReconnect: boolean | ((filters: Filter[]) => Filter[]) | undefined
+  public automaticallyAuth?: (relayURL: string) => null | ((event: EventTemplate) => Promise<VerifiedEvent>)
   public trustedRelayURLs: Set<string> = new Set()
 
   private _WebSocket?: typeof WebSocket
@@ -43,6 +47,7 @@ export class AbstractSimplePool {
     this._WebSocket = opts.websocketImplementation
     this.enablePing = opts.enablePing
     this.enableReconnect = opts.enableReconnect
+    this.automaticallyAuth = opts.automaticallyAuth
   }
 
   async ensureRelay(url: string, params?: { connectionTimeout?: number }): Promise<AbstractRelay> {
@@ -64,6 +69,14 @@ export class AbstractSimplePool {
       if (params?.connectionTimeout) relay.connectionTimeout = params.connectionTimeout
       this.relays.set(url, relay)
     }
+
+    if (this.automaticallyAuth) {
+      const authSignerFn = this.automaticallyAuth(url)
+      if (authSignerFn) {
+        relay.onauth = authSignerFn
+      }
+    }
+
     await relay.connect()
 
     return relay
@@ -77,8 +90,6 @@ export class AbstractSimplePool {
   }
 
   subscribe(relays: string[], filter: Filter, params: SubscribeManyParams): SubCloser {
-    params.onauth = params.onauth || params.doauth
-
     const request: { url: string; filter: Filter }[] = []
     for (let i = 0; i < relays.length; i++) {
       const url = normalizeURL(relays[i])
@@ -91,8 +102,6 @@ export class AbstractSimplePool {
   }
 
   subscribeMany(relays: string[], filter: Filter, params: SubscribeManyParams): SubCloser {
-    params.onauth = params.onauth || params.doauth
-
     const request: { url: string; filter: Filter }[] = []
     const uniqUrls: string[] = []
     for (let i = 0; i < relays.length; i++) {
@@ -107,8 +116,6 @@ export class AbstractSimplePool {
   }
 
   subscribeMap(requests: { url: string; filter: Filter }[], params: SubscribeManyParams): SubCloser {
-    params.onauth = params.onauth || params.doauth
-
     const grouped = new Map<string, Filter[]>()
     for (const req of requests) {
       const { url, filter } = req
@@ -221,10 +228,8 @@ export class AbstractSimplePool {
   subscribeEose(
     relays: string[],
     filter: Filter,
-    params: Pick<SubscribeManyParams, 'label' | 'id' | 'onevent' | 'onclose' | 'maxWait' | 'onauth' | 'doauth'>,
+    params: Pick<SubscribeManyParams, 'label' | 'id' | 'onevent' | 'onclose' | 'maxWait' | 'onauth'>,
   ): SubCloser {
-    params.onauth = params.onauth || params.doauth
-
     const subcloser = this.subscribe(relays, filter, {
       ...params,
       oneose() {
@@ -237,10 +242,8 @@ export class AbstractSimplePool {
   subscribeManyEose(
     relays: string[],
     filter: Filter,
-    params: Pick<SubscribeManyParams, 'label' | 'id' | 'onevent' | 'onclose' | 'maxWait' | 'onauth' | 'doauth'>,
+    params: Pick<SubscribeManyParams, 'label' | 'id' | 'onevent' | 'onclose' | 'maxWait' | 'onauth'>,
   ): SubCloser {
-    params.onauth = params.onauth || params.doauth
-
     const subcloser = this.subscribeMany(relays, filter, {
       ...params,
       oneose() {

abstract-relay.ts

@@ -32,6 +32,7 @@ export class AbstractRelay {
 
   public onclose: (() => void) | null = null
   public onnotice: (msg: string) => void = msg => console.debug(`NOTICE from ${this.url}: ${msg}`)
+  public onauth: undefined | ((evt: EventTemplate) => Promise<VerifiedEvent>)
 
   public baseEoseTimeout: number = 4400
   public connectionTimeout: number = 4400
@@ -158,12 +159,14 @@ export class AbstractRelay {
         }
         clearTimeout(this.connectionTimeoutHandle)
         this._connected = true
+
+        const isReconnection = this.reconnectAttempts > 0
         this.reconnectAttempts = 0
 
         // resubscribe to all open subscriptions
         for (const sub of this.openSubs.values()) {
           sub.eosed = false
-          if (typeof this.enableReconnect === 'function') {
+          if (isReconnection && typeof this.enableReconnect === 'function') {
             sub.filters = this.enableReconnect(sub.filters)
           }
           sub.fire()
@@ -205,14 +208,17 @@ export class AbstractRelay {
   private async waitForDummyReq() {
     return new Promise((resolve, _) => {
       // make a dummy request with expected empty eose reply
-      // ["REQ", "_", {"ids":["aaaa...aaaa"]}]
+      // ["REQ", "_", {"ids":["aaaa...aaaa"], "limit": 0}]
-      const sub = this.subscribe([{ ids: ['a'.repeat(64)] }], {
+      const sub = this.subscribe(
-        oneose: () => {
+        [{ ids: ['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'], limit: 0 }],
-          sub.close()
+        {
-          resolve(true)
+          oneose: () => {
+            sub.close()
+            resolve(true)
+          },
+          eoseTimeout: this.pingTimeout + 1000,
         },
-        eoseTimeout: this.pingTimeout + 1000,
+      )
-      })
     })
   }
 
@@ -256,6 +262,7 @@ export class AbstractRelay {
       return false
     }
 
+    // shortcut EVENT sub
     const subid = getSubscriptionId(json)
     if (subid) {
       const so = this.openSubs.get(subid as string)
@@ -331,11 +338,20 @@ export class AbstractRelay {
           so.close(data[2] as string)
           return
         }
-        case 'NOTICE':
+        case 'NOTICE': {
           this.onnotice(data[1] as string)
           return
+        }
         case 'AUTH': {
           this.challenge = data[1] as string
+          if (this.onauth) {
+            this.auth(this.onauth)
+          }
+          return
+        }
+        default: {
+          const so = this.openSubs.get(data[1])
+          so?.oncustom?.(data)
           return
         }
       }
@@ -405,9 +421,9 @@ export class AbstractRelay {
     filters: Filter[],
     params: Partial<SubscriptionParams> & { label?: string; id?: string },
   ): Subscription {
-    const subscription = this.prepareSubscription(filters, params)
+    const sub = this.prepareSubscription(filters, params)
-    subscription.fire()
+    sub.fire()
-    return subscription
+    return sub
   }
 
   public prepareSubscription(
@@ -463,10 +479,15 @@ export class Subscription {
   public oneose: (() => void) | undefined
   public onclose: ((reason: string) => void) | undefined
 
+  // will get any messages that have this subscription id as their second item and are not default standard
+  public oncustom: ((msg: string[]) => void) | undefined
+
   public eoseTimeout: number
   private eoseTimeoutHandle: ReturnType<typeof setTimeout> | undefined
 
   constructor(relay: AbstractRelay, id: string, filters: Filter[], params: SubscriptionParams) {
+    if (filters.length === 0) throw new Error("subscription can't be created with zero filters")
+
     this.relay = relay
     this.filters = filters
     this.id = id

build.js

@@ -7,7 +7,6 @@ const entryPoints = fs
   .filter(
     file =>
       file.endsWith('.ts') &&
-      file !== 'core.ts' &&
       file !== 'test-helpers.ts' &&
       file !== 'helpers.ts' &&
       file !== 'benchmarks.ts' &&

helpers.ts

@@ -1,17 +1,34 @@
 import { verifiedSymbol, type Event, type Nostr, VerifiedEvent } from './core.ts'
 
 export async function yieldThread() {
-  return new Promise<void>(resolve => {
+  return new Promise<void>((resolve, reject) => {
-    const ch = new MessageChannel()
+    try {
-    const handler = () => {
+      // Check if MessageChannel is available
-      // @ts-ignore (typescript thinks this property should be called `removeListener`, but in fact it's `removeEventListener`)
+      if (typeof MessageChannel !== 'undefined') {
-      ch.port1.removeEventListener('message', handler)
+        const ch = new MessageChannel()
-      resolve()
+        const handler = () => {
+          // @ts-ignore (typescript thinks this property should be called `removeListener`, but in fact it's `removeEventListener`)
+          ch.port1.removeEventListener('message', handler)
+          resolve()
+        }
+        // @ts-ignore (typescript thinks this property should be called `addListener`, but in fact it's `addEventListener`)
+        ch.port1.addEventListener('message', handler)
+        ch.port2.postMessage(0)
+        ch.port1.start()
+      } else {
+        if (typeof setImmediate !== 'undefined') {
+          setImmediate(resolve)
+        } else if (typeof setTimeout !== 'undefined') {
+          setTimeout(resolve, 0)
+        } else {
+          // Last resort - resolve immediately
+          resolve()
+        }
+      }
+    } catch (e) {
+      console.error('during yield: ', e)
+      reject(e)
     }
-    // @ts-ignore (typescript thinks this property should be called `addListener`, but in fact it's `addEventListener`)
-    ch.port1.addEventListener('message', handler)
-    ch.port2.postMessage(0)
-    ch.port1.start()
   })
 }
 

index.ts

@@ -24,6 +24,7 @@ export * as nip47 from './nip47.ts'
 export * as nip54 from './nip54.ts'
 export * as nip57 from './nip57.ts'
 export * as nip59 from './nip59.ts'
+export * as nip77 from './nip77.ts'
 export * as nip98 from './nip98.ts'
 
 export * as kinds from './kinds.ts'

jsr.json

@@ -1,6 +1,6 @@
 {
   "name": "@nostr/tools",
-  "version": "2.17.2",
+  "version": "2.19.0",
   "exports": {
     ".": "./index.ts",
     "./core": "./core.ts",

kinds.test.ts

@@ -18,7 +18,7 @@ test('kind classification', () => {
   expect(classifyKind(30000)).toBe('parameterized')
   expect(classifyKind(39999)).toBe('parameterized')
   expect(classifyKind(40000)).toBe('unknown')
-  expect(classifyKind(255)).toBe('unknown')
+  expect(classifyKind(255)).toBe('regular')
 })
 
 test('kind type guard', () => {

kinds.ts

@@ -2,12 +2,12 @@ import { NostrEvent, validateEvent } from './pure.ts'
 
 /** Events are **regular**, which means they're all expected to be stored by relays. */
 export function isRegularKind(kind: number): boolean {
-  return (1000 <= kind && kind < 10000) || [1, 2, 4, 5, 6, 7, 8, 16, 40, 41, 42, 43, 44].includes(kind)
+  return kind < 10000 && kind !== 0 && kind !== 3
 }
 
 /** Events are **replaceable**, which means that, for each combination of `pubkey` and `kind`, only the latest event is expected to (SHOULD) be stored by relays, older versions are expected to be discarded. */
 export function isReplaceableKind(kind: number): boolean {
-  return [0, 3].includes(kind) || (10000 <= kind && kind < 20000)
+  return kind === 0 || kind === 3 || (10000 <= kind && kind < 20000)
 }
 
 /** Events are **ephemeral**, which means they are not expected to be stored by relays. */

nip27.test.ts

@@ -1,5 +1,6 @@
 import { test, expect } from 'bun:test'
 import { parse } from './nip27.ts'
+import { NostrEvent } from './core.ts'
 
 test('first: parse simple content with 1 url and 1 nostr uri', () => {
   const content = `nostr:npub1hpslpc8c5sp3e2nhm2fr7swsfqpys5vyjar5dwpn7e7decps6r8qkcln63 check out my profile:nostr:npub1xtscya34g58tk0z605fvr788k263gsu6cy9x0mhnm87echrgufzsevkk5s; and this cool image https://images.com/image.jpg`
@@ -75,3 +76,40 @@ test('third: parse complex content with 4 nostr uris and 3 urls', () => {
     { type: 'url', url: 'https://example.com/docs' },
   ])
 })
+
+test('parse content with hashtags and emoji shortcodes', () => {
+  const event: NostrEvent = {
+    kind: 1,
+    tags: [
+      ['emoji', 'star', 'https://example.com/star.png'],
+      ['emoji', 'alpaca', 'https://example.com/alpaca.png'],
+    ],
+    content:
+      'hey nostr:npub1hpslpc8c5sp3e2nhm2fr7swsfqpys5vyjar5dwpn7e7decps6r8qkcln63 check out :alpaca::alpaca: #alpaca at wss://alpaca.com! :star:',
+    created_at: 1234567890,
+    pubkey: 'dummy',
+    id: 'dummy',
+    sig: 'dummy',
+  }
+  const blocks = Array.from(parse(event))
+
+  expect(blocks).toEqual([
+    { type: 'text', text: 'hey ' },
+    { type: 'reference', pointer: { pubkey: 'b861f0e0f8a4031caa77da923f41d04802485184974746b833f67cdce030d0ce' } },
+    { type: 'text', text: ' check out ' },
+    { type: 'emoji', shortcode: 'alpaca', url: 'https://example.com/alpaca.png' },
+    { type: 'emoji', shortcode: 'alpaca', url: 'https://example.com/alpaca.png' },
+    { type: 'text', text: ' ' },
+    { type: 'hashtag', value: 'alpaca' },
+    { type: 'text', text: ' at ' },
+    { type: 'relay', url: 'wss://alpaca.com/' },
+    { type: 'text', text: '! ' },
+    { type: 'emoji', shortcode: 'star', url: 'https://example.com/star.png' },
+  ])
+})
+
+test('emoji shortcodes are treated as text if no event tags', () => {
+  const blocks = Array.from(parse('hello :alpaca:'))
+
+  expect(blocks).toEqual([{ type: 'text', text: 'hello :alpaca:' }])
+})

nip27.ts

@@ -1,3 +1,4 @@
+import { NostrEvent } from './core.ts'
 import { AddressPointer, EventPointer, ProfilePointer, decode } from './nip19.ts'
 
 export type Block =
@@ -29,27 +30,67 @@ export type Block =
       type: 'audio'
       url: string
     }
+  | {
+      type: 'emoji'
+      shortcode: string
+      url: string
+    }
+  | {
+      type: 'hashtag'
+      value: string
+    }
 
 const noCharacter = /\W/m
 const noURLCharacter = /\W |\W$|$|,| /m
+const MAX_HASHTAG_LENGTH = 42
+
+export function* parse(content: string | NostrEvent): Iterable<Block> {
+  let emojis: { type: 'emoji'; shortcode: string; url: string }[] = []
+  if (typeof content !== 'string') {
+    for (let i = 0; i < content.tags.length; i++) {
+      const tag = content.tags[i]
+      if (tag[0] === 'emoji' && tag.length >= 3) {
+        emojis.push({ type: 'emoji', shortcode: tag[1], url: tag[2] })
+      }
+    }
+    content = content.content
+  }
 
-export function* parse(content: string): Iterable<Block> {
   const max = content.length
   let prevIndex = 0
   let index = 0
-  while (index < max) {
+  mainloop: while (index < max) {
-    let u = content.indexOf(':', index)
+    const u = content.indexOf(':', index)
-    if (u === -1) {
+    const h = content.indexOf('#', index)
+    if (u === -1 && h === -1) {
       // reached end
-      break
+      break mainloop
     }
 
-    if (content.substring(u - 5, u) === 'nostr') {
+    if (u === -1 || (h >= 0 && h < u)) {
-      const m = content.substring(u + 60).match(noCharacter)
+      // parse hashtag
+      if (h === 0 || content[h - 1] === ' ') {
+        const m = content.slice(h + 1, h + MAX_HASHTAG_LENGTH).match(noCharacter)
+        const end = m ? h + 1 + m.index! : max
+        yield { type: 'text', text: content.slice(prevIndex, h) }
+        yield { type: 'hashtag', value: content.slice(h + 1, end) }
+        index = end
+        prevIndex = index
+        continue mainloop
+      }
+
+      // ignore this, it is nothing
+      index = h + 1
+      continue mainloop
+    }
+
+    // otherwise parse things that have an ":"
+    if (content.slice(u - 5, u) === 'nostr') {
+      const m = content.slice(u + 60).match(noCharacter)
       const end = m ? u + 60 + m.index! : max
       try {
         let pointer: ProfilePointer | AddressPointer | EventPointer
-        let { data, type } = decode(content.substring(u + 1, end))
+        let { data, type } = decode(content.slice(u + 1, end))
 
         switch (type) {
           case 'npub':
@@ -65,89 +106,107 @@ export function* parse(content: string): Iterable<Block> {
         }
 
         if (prevIndex !== u - 5) {
-          yield { type: 'text', text: content.substring(prevIndex, u - 5) }
+          yield { type: 'text', text: content.slice(prevIndex, u - 5) }
         }
         yield { type: 'reference', pointer }
         index = end
         prevIndex = index
-        continue
+        continue mainloop
       } catch (_err) {
         // ignore this, not a valid nostr uri
         index = u + 1
-        continue
+        continue mainloop
       }
-    } else if (content.substring(u - 5, u) === 'https' || content.substring(u - 4, u) === 'http') {
+    } else if (content.slice(u - 5, u) === 'https' || content.slice(u - 4, u) === 'http') {
-      const m = content.substring(u + 4).match(noURLCharacter)
+      const m = content.slice(u + 4).match(noURLCharacter)
       const end = m ? u + 4 + m.index! : max
       const prefixLen = content[u - 1] === 's' ? 5 : 4
       try {
-        let url = new URL(content.substring(u - prefixLen, end))
+        let url = new URL(content.slice(u - prefixLen, end))
         if (url.hostname.indexOf('.') === -1) {
           throw new Error('invalid url')
         }
 
         if (prevIndex !== u - prefixLen) {
-          yield { type: 'text', text: content.substring(prevIndex, u - prefixLen) }
+          yield { type: 'text', text: content.slice(prevIndex, u - prefixLen) }
         }
 
-        if (/\.(png|jpe?g|gif|webp)$/i.test(url.pathname)) {
+        if (/\.(png|jpe?g|gif|webp|heic|svg)$/i.test(url.pathname)) {
           yield { type: 'image', url: url.toString() }
           index = end
           prevIndex = index
-          continue
+          continue mainloop
         }
-        if (/\.(mp4|avi|webm|mkv)$/i.test(url.pathname)) {
+        if (/\.(mp4|avi|webm|mkv|mov)$/i.test(url.pathname)) {
           yield { type: 'video', url: url.toString() }
           index = end
           prevIndex = index
-          continue
+          continue mainloop
         }
-        if (/\.(mp3|aac|ogg|opus)$/i.test(url.pathname)) {
+        if (/\.(mp3|aac|ogg|opus|wav|flac)$/i.test(url.pathname)) {
           yield { type: 'audio', url: url.toString() }
           index = end
           prevIndex = index
-          continue
+          continue mainloop
         }
 
         yield { type: 'url', url: url.toString() }
         index = end
         prevIndex = index
-        continue
+        continue mainloop
       } catch (_err) {
         // ignore this, not a valid url
         index = end + 1
-        continue
+        continue mainloop
       }
-    } else if (content.substring(u - 3, u) === 'wss' || content.substring(u - 2, u) === 'ws') {
+    } else if (content.slice(u - 3, u) === 'wss' || content.slice(u - 2, u) === 'ws') {
-      const m = content.substring(u + 4).match(noURLCharacter)
+      const m = content.slice(u + 4).match(noURLCharacter)
       const end = m ? u + 4 + m.index! : max
       const prefixLen = content[u - 1] === 's' ? 3 : 2
       try {
-        let url = new URL(content.substring(u - prefixLen, end))
+        let url = new URL(content.slice(u - prefixLen, end))
         if (url.hostname.indexOf('.') === -1) {
           throw new Error('invalid ws url')
         }
 
         if (prevIndex !== u - prefixLen) {
-          yield { type: 'text', text: content.substring(prevIndex, u - prefixLen) }
+          yield { type: 'text', text: content.slice(prevIndex, u - prefixLen) }
         }
         yield { type: 'relay', url: url.toString() }
         index = end
         prevIndex = index
-        continue
+        continue mainloop
       } catch (_err) {
         // ignore this, not a valid url
         index = end + 1
-        continue
+        continue mainloop
       }
     } else {
+      // try to parse an emoji shortcode
+      for (let e = 0; e < emojis.length; e++) {
+        const emoji = emojis[e]
+        if (
+          content[u + emoji.shortcode.length + 1] === ':' &&
+          content.slice(u + 1, u + emoji.shortcode.length + 1) === emoji.shortcode
+        ) {
+          // found an emoji
+          if (prevIndex !== u) {
+            yield { type: 'text', text: content.slice(prevIndex, u) }
+          }
+          yield emoji
+          index = u + emoji.shortcode.length + 2
+          prevIndex = index
+          continue mainloop
+        }
+      }
+
       // ignore this, it is nothing
       index = u + 1
-      continue
+      continue mainloop
     }
   }
 
   if (prevIndex !== max) {
-    yield { type: 'text', text: content.substring(prevIndex) }
+    yield { type: 'text', text: content.slice(prevIndex) }
   }
 }

nip77.test.ts

@@ -0,0 +1,114 @@
+import { describe, test, expect } from 'bun:test'
+import { NegentropySync, NegentropyStorageVector } from './nip77.ts'
+import { Relay } from './relay.ts'
+import { NostrEvent } from './core.ts'
+
+// const RELAY = 'ws://127.0.0.1:10547'
+const RELAY = 'wss://relay.damus.io'
+
+describe('NegentropySync', () => {
+  test('syncs events from ' + RELAY, async () => {
+    const relay = await Relay.connect(RELAY)
+
+    const storage = new NegentropyStorageVector()
+    storage.seal()
+    const filter = {
+      authors: ['3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d'],
+      kinds: [30617, 30618],
+    }
+
+    let ids1: string[] = []
+    const done1 = Promise.withResolvers<void>()
+    const sync1 = new NegentropySync(relay, storage, filter, {
+      onneed: (id: string) => {
+        ids1.push(id)
+      },
+      onclose: err => {
+        expect(err).toBeUndefined()
+        done1.resolve()
+      },
+    })
+
+    await sync1.start()
+    await done1.promise
+
+    expect(ids1.length).toBeGreaterThan(10)
+
+    sync1.close()
+
+    // fetch events
+    const events1: NostrEvent[] = []
+    const fetched = Promise.withResolvers()
+    const sub = relay.subscribe([{ ids: ids1 }], {
+      onevent(evt) {
+        events1.push(evt)
+      },
+      oneose() {
+        sub.close()
+        fetched.resolve()
+      },
+    })
+    await fetched.promise
+    expect(events1.map(evt => evt.id).sort()).toEqual(ids1.sort())
+
+    // Second sync with local events
+    await relay.connect()
+
+    const storage2 = new NegentropyStorageVector()
+    for (const evt of events1) {
+      storage2.insert(evt.created_at, evt.id)
+    }
+    storage2.seal()
+
+    let ids2: string[] = []
+    let done2 = Promise.withResolvers()
+    const sync2 = new NegentropySync(relay, storage2, filter, {
+      onneed: (id: string) => {
+        ids2.push(id)
+      },
+      onclose: err => {
+        expect(err).toBeUndefined()
+        done2.resolve()
+      },
+    })
+
+    await sync2.start()
+    await done2.promise
+
+    expect(ids2.length).toBe(0)
+
+    sync2.close()
+
+    // third sync with 4 events removed
+    const storage3 = new NegentropyStorageVector()
+
+    // shuffle
+    ids1.sort(() => Math.random() - 0.5)
+    const removedEvents = ids1.slice(0, 1 + Math.floor(Math.random() * ids1.length - 1))
+    for (const evt of events1) {
+      if (!removedEvents.includes(evt.id)) {
+        storage3.insert(evt.created_at, evt.id)
+      }
+    }
+    storage3.seal()
+
+    let ids3: string[] = []
+    const done3 = Promise.withResolvers()
+    const sync3 = new NegentropySync(relay, storage3, filter, {
+      onneed: (id: string) => {
+        ids3.push(id)
+      },
+      onclose: err => {
+        expect(err).toBeUndefined()
+        done3.resolve()
+      },
+    })
+
+    await sync3.start()
+    await done3.promise
+
+    expect(ids3.sort()).toEqual(removedEvents.sort())
+
+    sync3.close()
+  })
+})

nip77.ts

@@ -0,0 +1,607 @@
+import { bytesToHex, hexToBytes } from '@noble/ciphers/utils'
+import { Filter } from './filter.ts'
+import { AbstractRelay, Subscription } from './relay.ts'
+import { sha256 } from '@noble/hashes/sha256'
+
+// Negentropy implementation by Doug Hoyte
+const PROTOCOL_VERSION = 0x61 // Version 1
+const ID_SIZE = 32
+const FINGERPRINT_SIZE = 16
+
+const Mode = {
+  Skip: 0,
+  Fingerprint: 1,
+  IdList: 2,
+}
+
+class WrappedBuffer {
+  _raw: Uint8Array
+  length: number
+
+  constructor(buffer?: Uint8Array | number) {
+    if (typeof buffer === 'number') {
+      this._raw = new Uint8Array(buffer)
+      this.length = 0
+    } else if (buffer instanceof Uint8Array) {
+      this._raw = new Uint8Array(buffer)
+      this.length = buffer.length
+    } else {
+      this._raw = new Uint8Array(512)
+      this.length = 0
+    }
+  }
+
+  unwrap(): Uint8Array {
+    return this._raw.subarray(0, this.length)
+  }
+
+  get capacity(): number {
+    return this._raw.byteLength
+  }
+
+  extend(buf: Uint8Array | WrappedBuffer): void {
+    if (buf instanceof WrappedBuffer) buf = buf.unwrap()
+    if (typeof buf.length !== 'number') throw Error('bad length')
+    const targetSize = buf.length + this.length
+    if (this.capacity < targetSize) {
+      const oldRaw = this._raw
+      const newCapacity = Math.max(this.capacity * 2, targetSize)
+      this._raw = new Uint8Array(newCapacity)
+      this._raw.set(oldRaw)
+    }
+
+    this._raw.set(buf, this.length)
+    this.length += buf.length
+  }
+
+  shift(): number {
+    const first = this._raw[0]
+    this._raw = this._raw.subarray(1)
+    this.length--
+    return first
+  }
+
+  shiftN(n: number = 1): Uint8Array {
+    const firstSubarray = this._raw.subarray(0, n)
+    this._raw = this._raw.subarray(n)
+    this.length -= n
+    return firstSubarray
+  }
+}
+
+function decodeVarInt(buf: WrappedBuffer): number {
+  let res = 0
+
+  while (1) {
+    if (buf.length === 0) throw Error('parse ends prematurely')
+    let byte = buf.shift()
+    res = (res << 7) | (byte & 127)
+    if ((byte & 128) === 0) break
+  }
+
+  return res
+}
+
+function encodeVarInt(n: number): WrappedBuffer {
+  if (n === 0) return new WrappedBuffer(new Uint8Array([0]))
+
+  let o: number[] = []
+
+  while (n !== 0) {
+    o.push(n & 127)
+    n >>>= 7
+  }
+
+  o.reverse()
+
+  for (let i = 0; i < o.length - 1; i++) o[i] |= 128
+
+  return new WrappedBuffer(new Uint8Array(o))
+}
+
+function getByte(buf: WrappedBuffer): number {
+  return getBytes(buf, 1)[0]
+}
+
+function getBytes(buf: WrappedBuffer, n: number): Uint8Array {
+  if (buf.length < n) throw Error('parse ends prematurely')
+  return buf.shiftN(n)
+}
+
+class Accumulator {
+  buf!: Uint8Array
+
+  constructor() {
+    this.setToZero()
+  }
+
+  setToZero(): void {
+    this.buf = new Uint8Array(ID_SIZE)
+  }
+
+  add(otherBuf: Uint8Array): void {
+    let currCarry = 0,
+      nextCarry = 0
+    let p = new DataView(this.buf.buffer)
+    let po = new DataView(otherBuf.buffer)
+
+    for (let i = 0; i < 8; i++) {
+      let offset = i * 4
+      let orig = p.getUint32(offset, true)
+      let otherV = po.getUint32(offset, true)
+
+      let next = orig
+
+      next += currCarry
+      next += otherV
+      if (next > 0xffffffff) nextCarry = 1
+
+      p.setUint32(offset, next & 0xffffffff, true)
+      currCarry = nextCarry
+      nextCarry = 0
+    }
+  }
+
+  negate(): void {
+    let p = new DataView(this.buf.buffer)
+
+    for (let i = 0; i < 8; i++) {
+      let offset = i * 4
+      p.setUint32(offset, ~p.getUint32(offset, true))
+    }
+
+    let one = new Uint8Array(ID_SIZE)
+    one[0] = 1
+    this.add(one)
+  }
+
+  getFingerprint(n: number): Uint8Array {
+    let input = new WrappedBuffer()
+    input.extend(this.buf)
+    input.extend(encodeVarInt(n))
+
+    let hash = sha256(input.unwrap())
+    return hash.subarray(0, FINGERPRINT_SIZE)
+  }
+}
+
+export class NegentropyStorageVector {
+  items: { timestamp: number; id: Uint8Array }[]
+  sealed: boolean
+
+  constructor() {
+    this.items = []
+    this.sealed = false
+  }
+
+  insert(timestamp: number, id: string): void {
+    if (this.sealed) throw Error('already sealed')
+    const idb = hexToBytes(id)
+    if (idb.byteLength !== ID_SIZE) throw Error('bad id size for added item')
+    this.items.push({ timestamp, id: idb })
+  }
+
+  seal(): void {
+    if (this.sealed) throw Error('already sealed')
+    this.sealed = true
+
+    this.items.sort(itemCompare)
+
+    for (let i = 1; i < this.items.length; i++) {
+      if (itemCompare(this.items[i - 1], this.items[i]) === 0) throw Error('duplicate item inserted')
+    }
+  }
+
+  unseal(): void {
+    this.sealed = false
+  }
+
+  size(): number {
+    this._checkSealed()
+    return this.items.length
+  }
+
+  getItem(i: number): { timestamp: number; id: Uint8Array } {
+    this._checkSealed()
+    if (i >= this.items.length) throw Error('out of range')
+    return this.items[i]
+  }
+
+  iterate(begin: number, end: number, cb: (item: { timestamp: number; id: Uint8Array }, i: number) => boolean): void {
+    this._checkSealed()
+    this._checkBounds(begin, end)
+
+    for (let i = begin; i < end; ++i) {
+      if (!cb(this.items[i], i)) break
+    }
+  }
+
+  findLowerBound(begin: number, end: number, bound: { timestamp: number; id: Uint8Array }): number {
+    this._checkSealed()
+    this._checkBounds(begin, end)
+
+    return this._binarySearch(this.items, begin, end, a => itemCompare(a, bound) < 0)
+  }
+
+  fingerprint(begin: number, end: number): Uint8Array {
+    let out = new Accumulator()
+    out.setToZero()
+
+    this.iterate(begin, end, item => {
+      out.add(item.id)
+      return true
+    })
+
+    return out.getFingerprint(end - begin)
+  }
+
+  _checkSealed(): void {
+    if (!this.sealed) throw Error('not sealed')
+  }
+
+  _checkBounds(begin: number, end: number): void {
+    if (begin > end || end > this.items.length) throw Error('bad range')
+  }
+
+  _binarySearch(
+    arr: { timestamp: number; id: Uint8Array }[],
+    first: number,
+    last: number,
+    cmp: (a: { timestamp: number; id: Uint8Array }) => boolean,
+  ): number {
+    let count = last - first
+
+    while (count > 0) {
+      let it = first
+      let step = Math.floor(count / 2)
+      it += step
+
+      if (cmp(arr[it])) {
+        first = ++it
+        count -= step + 1
+      } else {
+        count = step
+      }
+    }
+
+    return first
+  }
+}
+
+export class Negentropy {
+  storage: NegentropyStorageVector
+  frameSizeLimit: number
+  lastTimestampIn: number
+  lastTimestampOut: number
+
+  constructor(storage: NegentropyStorageVector, frameSizeLimit: number = 60_000) {
+    if (frameSizeLimit < 4096) throw Error('frameSizeLimit too small')
+
+    this.storage = storage
+    this.frameSizeLimit = frameSizeLimit
+
+    this.lastTimestampIn = 0
+    this.lastTimestampOut = 0
+  }
+
+  _bound(timestamp: number, id?: Uint8Array): { timestamp: number; id: Uint8Array } {
+    return { timestamp, id: id || new Uint8Array(0) }
+  }
+
+  initiate(): string {
+    let output = new WrappedBuffer()
+    output.extend(new Uint8Array([PROTOCOL_VERSION]))
+    this.splitRange(0, this.storage.size(), this._bound(Number.MAX_VALUE), output)
+    return bytesToHex(output.unwrap())
+  }
+
+  reconcile(queryMsg: string, onhave?: (id: string) => void, onneed?: (id: string) => void): string | null {
+    const query = new WrappedBuffer(hexToBytes(queryMsg))
+
+    this.lastTimestampIn = this.lastTimestampOut = 0 // reset for each message
+
+    let fullOutput = new WrappedBuffer()
+    fullOutput.extend(new Uint8Array([PROTOCOL_VERSION]))
+
+    let protocolVersion = getByte(query)
+    if (protocolVersion < 0x60 || protocolVersion > 0x6f) throw Error('invalid negentropy protocol version byte')
+    if (protocolVersion !== PROTOCOL_VERSION) {
+      throw Error('unsupported negentropy protocol version requested: ' + (protocolVersion - 0x60))
+    }
+
+    let storageSize = this.storage.size()
+    let prevBound = this._bound(0)
+    let prevIndex = 0
+    let skip = false
+
+    while (query.length !== 0) {
+      let o = new WrappedBuffer()
+
+      let doSkip = () => {
+        if (skip) {
+          skip = false
+          o.extend(this.encodeBound(prevBound))
+          o.extend(encodeVarInt(Mode.Skip))
+        }
+      }
+
+      let currBound = this.decodeBound(query)
+      let mode = decodeVarInt(query)
+
+      let lower = prevIndex
+      let upper = this.storage.findLowerBound(prevIndex, storageSize, currBound)
+
+      if (mode === Mode.Skip) {
+        skip = true
+      } else if (mode === Mode.Fingerprint) {
+        let theirFingerprint = getBytes(query, FINGERPRINT_SIZE)
+        let ourFingerprint = this.storage.fingerprint(lower, upper)
+
+        if (compareUint8Array(theirFingerprint, ourFingerprint) !== 0) {
+          doSkip()
+          this.splitRange(lower, upper, currBound, o)
+        } else {
+          skip = true
+        }
+      } else if (mode === Mode.IdList) {
+        let numIds = decodeVarInt(query)
+
+        let theirElems: { [key: string]: Uint8Array } = {} // stringified Uint8Array -> original Uint8Array (or hex)
+        for (let i = 0; i < numIds; i++) {
+          let e = getBytes(query, ID_SIZE)
+          theirElems[bytesToHex(e)] = e
+        }
+
+        skip = true
+        this.storage.iterate(lower, upper, item => {
+          let k = item.id
+          const id = bytesToHex(k)
+
+          if (!theirElems[id]) {
+            // ID exists on our side, but not their side
+            onhave?.(id)
+          } else {
+            // ID exists on both sides
+            delete theirElems[bytesToHex(k)]
+          }
+
+          return true
+        })
+
+        if (onneed) {
+          for (let v of Object.values(theirElems)) {
+            // ID exists on their side, but not our side
+            onneed(bytesToHex(v))
+          }
+        }
+      } else {
+        throw Error('unexpected mode')
+      }
+
+      if (this.exceededFrameSizeLimit(fullOutput.length + o.length)) {
+        // frameSizeLimit exceeded: stop range processing and return a fingerprint for the remaining range
+        let remainingFingerprint = this.storage.fingerprint(upper, storageSize)
+
+        fullOutput.extend(this.encodeBound(this._bound(Number.MAX_VALUE)))
+        fullOutput.extend(encodeVarInt(Mode.Fingerprint))
+        fullOutput.extend(remainingFingerprint)
+        break
+      } else {
+        fullOutput.extend(o)
+      }
+
+      prevIndex = upper
+      prevBound = currBound
+    }
+
+    return fullOutput.length === 1 ? null : bytesToHex(fullOutput.unwrap())
+  }
+
+  splitRange(lower: number, upper: number, upperBound: { timestamp: number; id: Uint8Array }, o: WrappedBuffer) {
+    let numElems = upper - lower
+    let buckets = 16
+
+    if (numElems < buckets * 2) {
+      o.extend(this.encodeBound(upperBound))
+      o.extend(encodeVarInt(Mode.IdList))
+
+      o.extend(encodeVarInt(numElems))
+      this.storage.iterate(lower, upper, item => {
+        o.extend(item.id)
+        return true
+      })
+    } else {
+      let itemsPerBucket = Math.floor(numElems / buckets)
+      let bucketsWithExtra = numElems % buckets
+      let curr = lower
+
+      for (let i = 0; i < buckets; i++) {
+        let bucketSize = itemsPerBucket + (i < bucketsWithExtra ? 1 : 0)
+        let ourFingerprint = this.storage.fingerprint(curr, curr + bucketSize)
+        curr += bucketSize
+
+        let nextBound: { timestamp: number; id: Uint8Array }
+
+        if (curr === upper) {
+          nextBound = upperBound
+        } else {
+          let prevItem: { timestamp: number; id: Uint8Array } | undefined
+          let currItem: { timestamp: number; id: Uint8Array } | undefined
+
+          this.storage.iterate(curr - 1, curr + 1, (item, index) => {
+            if (index === curr - 1) prevItem = item
+            else currItem = item
+            return true
+          })
+
+          nextBound = this.getMinimalBound(prevItem!, currItem!)
+        }
+
+        o.extend(this.encodeBound(nextBound))
+        o.extend(encodeVarInt(Mode.Fingerprint))
+        o.extend(ourFingerprint)
+      }
+    }
+  }
+
+  exceededFrameSizeLimit(n: number): boolean {
+    return n > this.frameSizeLimit - 200
+  }
+
+  // Decoding
+  decodeTimestampIn(encoded: WrappedBuffer): number {
+    let timestamp = decodeVarInt(encoded)
+    timestamp = timestamp === 0 ? Number.MAX_VALUE : timestamp - 1
+    if (this.lastTimestampIn === Number.MAX_VALUE || timestamp === Number.MAX_VALUE) {
+      this.lastTimestampIn = Number.MAX_VALUE
+      return Number.MAX_VALUE
+    }
+    timestamp += this.lastTimestampIn
+    this.lastTimestampIn = timestamp
+    return timestamp
+  }
+
+  decodeBound(encoded: WrappedBuffer): { timestamp: number; id: Uint8Array } {
+    let timestamp = this.decodeTimestampIn(encoded)
+    let len = decodeVarInt(encoded)
+    if (len > ID_SIZE) throw Error('bound key too long')
+    let id = getBytes(encoded, len)
+    return { timestamp, id }
+  }
+
+  // Encoding
+  encodeTimestampOut(timestamp: number): WrappedBuffer {
+    if (timestamp === Number.MAX_VALUE) {
+      this.lastTimestampOut = Number.MAX_VALUE
+      return encodeVarInt(0)
+    }
+
+    let temp = timestamp
+    timestamp -= this.lastTimestampOut
+    this.lastTimestampOut = temp
+    return encodeVarInt(timestamp + 1)
+  }
+
+  encodeBound(key: { timestamp: number; id: Uint8Array }): WrappedBuffer {
+    let output = new WrappedBuffer()
+
+    output.extend(this.encodeTimestampOut(key.timestamp))
+    output.extend(encodeVarInt(key.id.length))
+    output.extend(key.id)
+
+    return output
+  }
+
+  getMinimalBound(
+    prev: { timestamp: number; id: Uint8Array },
+    curr: { timestamp: number; id: Uint8Array },
+  ): { timestamp: number; id: Uint8Array } {
+    if (curr.timestamp !== prev.timestamp) {
+      return this._bound(curr.timestamp)
+    } else {
+      let sharedPrefixBytes = 0
+      let currKey = curr.id
+      let prevKey = prev.id
+
+      for (let i = 0; i < ID_SIZE; i++) {
+        if (currKey[i] !== prevKey[i]) break
+        sharedPrefixBytes++
+      }
+
+      return this._bound(curr.timestamp, curr.id.subarray(0, sharedPrefixBytes + 1))
+    }
+  }
+}
+
+function compareUint8Array(a: Uint8Array, b: Uint8Array): number {
+  for (let i = 0; i < a.byteLength; i++) {
+    if (a[i] < b[i]) return -1
+    if (a[i] > b[i]) return 1
+  }
+
+  if (a.byteLength > b.byteLength) return 1
+  if (a.byteLength < b.byteLength) return -1
+
+  return 0
+}
+
+function itemCompare(a: { timestamp: number; id: Uint8Array }, b: { timestamp: number; id: Uint8Array }): number {
+  if (a.timestamp === b.timestamp) {
+    return compareUint8Array(a.id, b.id)
+  }
+
+  return a.timestamp - b.timestamp
+}
+
+export class NegentropySync {
+  relay: AbstractRelay
+  storage: NegentropyStorageVector
+  private neg: Negentropy
+  private filter: Filter
+  private subscription: Subscription
+  private onhave?: (id: string) => void
+  private onneed?: (id: string) => void
+
+  constructor(
+    relay: AbstractRelay,
+    storage: NegentropyStorageVector,
+    filter: Filter,
+    params: {
+      label?: string
+      onhave?: (id: string) => void
+      onneed?: (id: string) => void
+      onclose?: (errReason?: string) => void
+    } = {},
+  ) {
+    this.relay = relay
+    this.storage = storage
+    this.neg = new Negentropy(storage)
+    this.onhave = params.onhave
+    this.onneed = params.onneed
+    this.filter = filter
+
+    // we prepare a subscription with an empty filter, but it will not be used
+    this.subscription = this.relay.prepareSubscription([{}], { label: params.label || 'negentropy' })
+    this.subscription.oncustom = (data: string[]) => {
+      switch (data[0]) {
+        case 'NEG-MSG': {
+          if (data.length < 3) {
+            console.warn(`got invalid NEG-MSG from ${this.relay.url}: ${data}`)
+          }
+          try {
+            const response = this.neg.reconcile(data[2], this.onhave, this.onneed)
+            if (response) {
+              this.relay.send(`["NEG-MSG", "${this.subscription.id}", "${response}"]`)
+            } else {
+              this.close()
+              params.onclose?.()
+            }
+          } catch (error) {
+            console.error('negentropy reconcile error:', error)
+            params?.onclose?.(`reconcile error: ${error}`)
+          }
+          break
+        }
+        case 'NEG-CLOSE': {
+          const reason = data[2]
+          console.warn('negentropy error:', reason)
+          params.onclose?.(reason)
+          break
+        }
+        case 'NEG-ERR': {
+          params.onclose?.()
+        }
+      }
+    }
+  }
+
+  async start(): Promise<void> {
+    const initMsg = this.neg.initiate()
+    this.relay.send(`["NEG-OPEN","${this.subscription.id}",${JSON.stringify(this.filter)},"${initMsg}"]`)
+  }
+
+  close(): void {
+    this.relay.send(`["NEG-CLOSE","${this.subscription.id}"]`)
+    this.subscription.close()
+  }
+}

package.json

@@ -1,7 +1,7 @@
 {
   "type": "module",
   "name": "nostr-tools",
-  "version": "2.17.2",
+  "version": "2.19.0",
   "description": "Tools for making a Nostr client.",
   "repository": {
     "type": "git",

utils.test.ts

@@ -1,6 +1,12 @@
 import { describe, test, expect } from 'bun:test'
 import { buildEvent } from './test-helpers.ts'
-import { Queue, insertEventIntoAscendingList, insertEventIntoDescendingList, binarySearch } from './utils.ts'
+import {
+  Queue,
+  insertEventIntoAscendingList,
+  insertEventIntoDescendingList,
+  binarySearch,
+  normalizeURL,
+} from './utils.ts'
 
 import type { Event } from './core.ts'
 
@@ -263,3 +269,43 @@ test('binary search', () => {
   expect(binarySearch(['a', 'b', 'd', 'e'], b => ('a' < b ? -1 : 'a' === b ? 0 : 1))).toEqual([0, true])
   expect(binarySearch(['a', 'b', 'd', 'e'], b => ('[' < b ? -1 : '[' === b ? 0 : 1))).toEqual([0, false])
 })
+
+describe('normalizeURL', () => {
+  test('normalizes wss:// URLs', () => {
+    expect(normalizeURL('wss://example.com')).toBe('wss://example.com/')
+    expect(normalizeURL('wss://example.com/')).toBe('wss://example.com/')
+    expect(normalizeURL('wss://example.com//path')).toBe('wss://example.com/path')
+    expect(normalizeURL('wss://example.com:443')).toBe('wss://example.com/')
+  })
+
+  test('normalizes https:// URLs', () => {
+    expect(normalizeURL('https://example.com')).toBe('wss://example.com/')
+    expect(normalizeURL('https://example.com/')).toBe('wss://example.com/')
+    expect(normalizeURL('http://example.com//path')).toBe('ws://example.com/path')
+  })
+
+  test('normalizes ws:// URLs', () => {
+    expect(normalizeURL('ws://example.com')).toBe('ws://example.com/')
+    expect(normalizeURL('ws://example.com/')).toBe('ws://example.com/')
+    expect(normalizeURL('ws://example.com//path')).toBe('ws://example.com/path')
+    expect(normalizeURL('ws://example.com:80')).toBe('ws://example.com/')
+  })
+
+  test('adds wss:// to URLs without scheme', () => {
+    expect(normalizeURL('example.com')).toBe('wss://example.com/')
+    expect(normalizeURL('example.com/')).toBe('wss://example.com/')
+    expect(normalizeURL('example.com//path')).toBe('wss://example.com/path')
+  })
+
+  test('handles query parameters', () => {
+    expect(normalizeURL('wss://example.com?z=1&a=2')).toBe('wss://example.com/?a=2&z=1')
+  })
+
+  test('removes hash', () => {
+    expect(normalizeURL('wss://example.com#hash')).toBe('wss://example.com/')
+  })
+
+  test('throws on invalid URL', () => {
+    expect(() => normalizeURL('http://')).toThrow('Invalid URL: http://')
+  })
+})

utils.ts

@@ -9,6 +9,8 @@ export function normalizeURL(url: string): string {
   try {
     if (url.indexOf('://') === -1) url = 'wss://' + url
     let p = new URL(url)
+    if (p.protocol === 'http:') p.protocol = 'ws:'
+    else if (p.protocol === 'https:') p.protocol = 'wss:'
     p.pathname = p.pathname.replace(/\/+/g, '/')
     if (p.pathname.endsWith('/')) p.pathname = p.pathname.slice(0, -1)
     if ((p.port === '80' && p.protocol === 'ws:') || (p.port === '443' && p.protocol === 'wss:')) p.port = ''