update README with new enableReconnect() behavior.

prevent call stacks from building up

README.md

@@ -160,16 +160,7 @@ Using both `enablePing: true` and `enableReconnect: true` is recommended as it w
 const pool = new SimplePool({ enablePing: true, enableReconnect: true })
 ```
 
-The `enableReconnect` option can also be a callback function which will receive the current subscription filters and should return a new set of filters. This is useful if you want to modify the subscription on reconnect, for example, to update the `since` parameter to fetch only new events.
+When reconnecting, all existing subscriptions will have their filters automatically updated with `since:` set to the timestamp of the last event received on them `+1`, then restarted.
-
-```js
-const pool = new SimplePool({
-  enableReconnect: (filters) => {
-    const newSince = Math.floor(Date.now() / 1000)
-    return filters.map(filter => ({ ...filter, since: newSince }))
-  }
-})
-```
 
 ### Parsing references (mentions) from a content based on NIP-27
 

abstract-pool.ts

@@ -14,14 +14,17 @@ import { alwaysTrue } from './helpers.ts'
 
 export type SubCloser = { close: (reason?: string) => void }
 
-export type AbstractPoolConstructorOptions = AbstractRelayConstructorOptions & {}
+export type AbstractPoolConstructorOptions = AbstractRelayConstructorOptions & {
+  // automaticallyAuth takes a relay URL and should return null
+  // in case that relay shouldn't be authenticated against
+  // or a function to sign the AUTH event template otherwise (that function may still throw in case of failure)
+  automaticallyAuth?: (relayURL: string) => null | ((event: EventTemplate) => Promise<VerifiedEvent>)
+}
 
 export type SubscribeManyParams = Omit<SubscriptionParams, 'onclose'> & {
   maxWait?: number
   onclose?: (reasons: string[]) => void
   onauth?: (event: EventTemplate) => Promise<VerifiedEvent>
-  // Deprecated: use onauth instead
-  doauth?: (event: EventTemplate) => Promise<VerifiedEvent>
   id?: string
   label?: string
 }
@@ -33,7 +36,8 @@ export class AbstractSimplePool {
 
   public verifyEvent: Nostr['verifyEvent']
   public enablePing: boolean | undefined
-  public enableReconnect: boolean | ((filters: Filter[]) => Filter[]) | undefined
+  public enableReconnect: boolean
+  public automaticallyAuth?: (relayURL: string) => null | ((event: EventTemplate) => Promise<VerifiedEvent>)
   public trustedRelayURLs: Set<string> = new Set()
 
   private _WebSocket?: typeof WebSocket
@@ -42,7 +46,8 @@ export class AbstractSimplePool {
     this.verifyEvent = opts.verifyEvent
     this._WebSocket = opts.websocketImplementation
     this.enablePing = opts.enablePing
-    this.enableReconnect = opts.enableReconnect
+    this.enableReconnect = opts.enableReconnect || false
+    this.automaticallyAuth = opts.automaticallyAuth
   }
 
   async ensureRelay(url: string, params?: { connectionTimeout?: number }): Promise<AbstractRelay> {
@@ -64,6 +69,14 @@ export class AbstractSimplePool {
       if (params?.connectionTimeout) relay.connectionTimeout = params.connectionTimeout
       this.relays.set(url, relay)
     }
+
+    if (this.automaticallyAuth) {
+      const authSignerFn = this.automaticallyAuth(url)
+      if (authSignerFn) {
+        relay.onauth = authSignerFn
+      }
+    }
+
     await relay.connect()
 
     return relay
@@ -77,8 +90,6 @@ export class AbstractSimplePool {
   }
 
   subscribe(relays: string[], filter: Filter, params: SubscribeManyParams): SubCloser {
-    params.onauth = params.onauth || params.doauth
-
     const request: { url: string; filter: Filter }[] = []
     for (let i = 0; i < relays.length; i++) {
       const url = normalizeURL(relays[i])
@@ -91,8 +102,6 @@ export class AbstractSimplePool {
   }
 
   subscribeMany(relays: string[], filter: Filter, params: SubscribeManyParams): SubCloser {
-    params.onauth = params.onauth || params.doauth
-
     const request: { url: string; filter: Filter }[] = []
     const uniqUrls: string[] = []
     for (let i = 0; i < relays.length; i++) {
@@ -107,8 +116,6 @@ export class AbstractSimplePool {
   }
 
   subscribeMap(requests: { url: string; filter: Filter }[], params: SubscribeManyParams): SubCloser {
-    params.onauth = params.onauth || params.doauth
-
     const grouped = new Map<string, Filter[]>()
     for (const req of requests) {
       const { url, filter } = req
@@ -221,10 +228,8 @@ export class AbstractSimplePool {
   subscribeEose(
     relays: string[],
     filter: Filter,
-    params: Pick<SubscribeManyParams, 'label' | 'id' | 'onevent' | 'onclose' | 'maxWait' | 'onauth' | 'doauth'>,
+    params: Pick<SubscribeManyParams, 'label' | 'id' | 'onevent' | 'onclose' | 'maxWait' | 'onauth'>,
   ): SubCloser {
-    params.onauth = params.onauth || params.doauth
-
     const subcloser = this.subscribe(relays, filter, {
       ...params,
       oneose() {
@@ -237,10 +242,8 @@ export class AbstractSimplePool {
   subscribeManyEose(
     relays: string[],
     filter: Filter,
-    params: Pick<SubscribeManyParams, 'label' | 'id' | 'onevent' | 'onclose' | 'maxWait' | 'onauth' | 'doauth'>,
+    params: Pick<SubscribeManyParams, 'label' | 'id' | 'onevent' | 'onclose' | 'maxWait' | 'onauth'>,
   ): SubCloser {
-    params.onauth = params.onauth || params.doauth
-
     const subcloser = this.subscribeMany(relays, filter, {
       ...params,
       oneose() {

abstract-relay.ts

@@ -16,7 +16,7 @@ export type AbstractRelayConstructorOptions = {
   verifyEvent: Nostr['verifyEvent']
   websocketImplementation?: typeof WebSocket
   enablePing?: boolean
-  enableReconnect?: boolean | ((filters: Filter[]) => Filter[])
+  enableReconnect?: boolean
 }
 
 export class SendingOnClosedConnection extends Error {
@@ -32,19 +32,20 @@ export class AbstractRelay {
 
   public onclose: (() => void) | null = null
   public onnotice: (msg: string) => void = msg => console.debug(`NOTICE from ${this.url}: ${msg}`)
+  public onauth: undefined | ((evt: EventTemplate) => Promise<VerifiedEvent>)
 
   public baseEoseTimeout: number = 4400
   public connectionTimeout: number = 4400
   public publishTimeout: number = 4400
-  public pingFrequency: number = 20000
+  public pingFrequency: number = 29000
   public pingTimeout: number = 20000
   public resubscribeBackoff: number[] = [10000, 10000, 10000, 20000, 20000, 30000, 60000]
   public openSubs: Map<string, Subscription> = new Map()
   public enablePing: boolean | undefined
-  public enableReconnect: boolean | ((filters: Filter[]) => Filter[])
+  public enableReconnect: boolean
   private connectionTimeoutHandle: ReturnType<typeof setTimeout> | undefined
   private reconnectTimeoutHandle: ReturnType<typeof setTimeout> | undefined
-  private pingTimeoutHandle: ReturnType<typeof setTimeout> | undefined
+  private pingIntervalHandle: ReturnType<typeof setInterval> | undefined
   private reconnectAttempts: number = 0
   private closedIntentionally: boolean = false
 
@@ -110,9 +111,9 @@ export class AbstractRelay {
   }
 
   private handleHardClose(reason: string) {
-    if (this.pingTimeoutHandle) {
+    if (this.pingIntervalHandle) {
-      clearTimeout(this.pingTimeoutHandle)
+      clearInterval(this.pingIntervalHandle)
-      this.pingTimeoutHandle = undefined
+      this.pingIntervalHandle = undefined
     }
 
     this._connected = false
@@ -158,19 +159,25 @@ export class AbstractRelay {
         }
         clearTimeout(this.connectionTimeoutHandle)
         this._connected = true
+
+        const isReconnection = this.reconnectAttempts > 0
         this.reconnectAttempts = 0
 
         // resubscribe to all open subscriptions
         for (const sub of this.openSubs.values()) {
           sub.eosed = false
-          if (typeof this.enableReconnect === 'function') {
+          if (isReconnection) {
-            sub.filters = this.enableReconnect(sub.filters)
+            for (let f = 0; f < sub.filters.length; f++) {
+              if (sub.lastEmitted) {
+                sub.filters[f].since = sub.lastEmitted + 1
+              }
+            }
           }
           sub.fire()
         }
 
         if (this.enablePing) {
-          this.pingpong()
+          this.pingIntervalHandle = setInterval(() => this.pingpong(), this.pingFrequency)
         }
         resolve()
       }
@@ -202,17 +209,30 @@ export class AbstractRelay {
     })
   }
 
-  private async waitForDummyReq() {
+  private waitForDummyReq() {
-    return new Promise((resolve, _) => {
+    return new Promise((resolve, reject) => {
+      if (!this.connectionPromise) return reject(new Error(`no connection to ${this.url}, can't ping`))
+
       // make a dummy request with expected empty eose reply
-      // ["REQ", "_", {"ids":["aaaa...aaaa"]}]
+      // ["REQ", "_", {"ids":["aaaa...aaaa"], "limit": 0}]
-      const sub = this.subscribe([{ ids: ['a'.repeat(64)] }], {
+      try {
+        const sub = this.subscribe(
+          [{ ids: ['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'], limit: 0 }],
+          {
             oneose: () => {
+              resolve(true)
               sub.close()
+            },
+            onclose() {
+              // if we get a CLOSED it's because the relay is alive
               resolve(true)
             },
             eoseTimeout: this.pingTimeout + 1000,
-      })
+          },
+        )
+      } catch (err) {
+        reject(err)
+      }
     })
   }
 
@@ -227,10 +247,8 @@ export class AbstractRelay {
         this.ws && this.ws.ping && (this.ws as any).once ? this.waitForPingPong() : this.waitForDummyReq(),
         new Promise(res => setTimeout(() => res(false), this.pingTimeout)),
       ])
-      if (result) {
+
-        // schedule another pingpong
+      if (!result) {
-        this.pingTimeoutHandle = setTimeout(() => this.pingpong(), this.pingFrequency)
-      } else {
         // pingpong closing socket
         if (this.ws?.readyState === this._WebSocket.OPEN) {
           this.ws?.close()
@@ -293,6 +311,7 @@ export class AbstractRelay {
           if (this.verifyEvent(event) && matchFilters(so.filters, event)) {
             so.onevent(event)
           }
+          if (!so.lastEmitted || so.lastEmitted < event.created_at) so.lastEmitted = event.created_at
           return
         }
         case 'COUNT': {
@@ -338,6 +357,9 @@ export class AbstractRelay {
         }
         case 'AUTH': {
           this.challenge = data[1] as string
+          if (this.onauth) {
+            this.auth(this.onauth)
+          }
           return
         }
         default: {
@@ -434,9 +456,9 @@ export class AbstractRelay {
       clearTimeout(this.reconnectTimeoutHandle)
       this.reconnectTimeoutHandle = undefined
     }
-    if (this.pingTimeoutHandle) {
+    if (this.pingIntervalHandle) {
-      clearTimeout(this.pingTimeoutHandle)
+      clearInterval(this.pingIntervalHandle)
-      this.pingTimeoutHandle = undefined
+      this.pingIntervalHandle = undefined
     }
     this.closeAllSubscriptions('relay connection closed by us')
     this._connected = false
@@ -460,6 +482,7 @@ export class Subscription {
   public readonly relay: AbstractRelay
   public readonly id: string
 
+  public lastEmitted: number | undefined
   public closed: boolean = false
   public eosed: boolean = false
   public filters: Filter[]

jsr.json

@@ -1,6 +1,6 @@
 {
   "name": "@nostr/tools",
-  "version": "2.18.0",
+  "version": "2.19.2",
   "exports": {
     ".": "./index.ts",
     "./core": "./core.ts",

kinds.test.ts

@@ -18,7 +18,7 @@ test('kind classification', () => {
   expect(classifyKind(30000)).toBe('parameterized')
   expect(classifyKind(39999)).toBe('parameterized')
   expect(classifyKind(40000)).toBe('unknown')
-  expect(classifyKind(255)).toBe('unknown')
+  expect(classifyKind(255)).toBe('regular')
 })
 
 test('kind type guard', () => {

kinds.ts

@@ -2,12 +2,12 @@ import { NostrEvent, validateEvent } from './pure.ts'
 
 /** Events are **regular**, which means they're all expected to be stored by relays. */
 export function isRegularKind(kind: number): boolean {
-  return (1000 <= kind && kind < 10000) || [1, 2, 4, 5, 6, 7, 8, 16, 40, 41, 42, 43, 44].includes(kind)
+  return kind < 10000 && kind !== 0 && kind !== 3
 }
 
 /** Events are **replaceable**, which means that, for each combination of `pubkey` and `kind`, only the latest event is expected to (SHOULD) be stored by relays, older versions are expected to be discarded. */
 export function isReplaceableKind(kind: number): boolean {
-  return [0, 3].includes(kind) || (10000 <= kind && kind < 20000)
+  return kind === 0 || kind === 3 || (10000 <= kind && kind < 20000)
 }
 
 /** Events are **ephemeral**, which means they are not expected to be stored by relays. */

nip27.test.ts

@@ -107,3 +107,9 @@ test('parse content with hashtags and emoji shortcodes', () => {
     { type: 'emoji', shortcode: 'star', url: 'https://example.com/star.png' },
   ])
 })
+
+test('emoji shortcodes are treated as text if no event tags', () => {
+  const blocks = Array.from(parse('hello :alpaca:'))
+
+  expect(blocks).toEqual([{ type: 'text', text: 'hello :alpaca:' }])
+})

nip27.ts

@@ -131,19 +131,19 @@ export function* parse(content: string | NostrEvent): Iterable<Block> {
           yield { type: 'text', text: content.slice(prevIndex, u - prefixLen) }
         }
 
-        if (/\.(png|jpe?g|gif|webp)$/i.test(url.pathname)) {
+        if (/\.(png|jpe?g|gif|webp|heic|svg)$/i.test(url.pathname)) {
           yield { type: 'image', url: url.toString() }
           index = end
           prevIndex = index
           continue mainloop
         }
-        if (/\.(mp4|avi|webm|mkv)$/i.test(url.pathname)) {
+        if (/\.(mp4|avi|webm|mkv|mov)$/i.test(url.pathname)) {
           yield { type: 'video', url: url.toString() }
           index = end
           prevIndex = index
           continue mainloop
         }
-        if (/\.(mp3|aac|ogg|opus)$/i.test(url.pathname)) {
+        if (/\.(mp3|aac|ogg|opus|wav|flac)$/i.test(url.pathname)) {
           yield { type: 'audio', url: url.toString() }
           index = end
           prevIndex = index

package.json

@@ -1,7 +1,7 @@
 {
   "type": "module",
   "name": "nostr-tools",
-  "version": "2.18.0",
+  "version": "2.19.2",
   "description": "Tools for making a Nostr client.",
   "repository": {
     "type": "git",

pool.test.ts

@@ -306,12 +306,9 @@ test('reconnect on disconnect in pool', async () => {
 
 test('reconnect with filter update in pool', async () => {
   const mockRelay = mockRelays[0]
-  const newSince = Math.floor(Date.now() / 1000)
   pool = new SimplePool({
     enablePing: true,
-    enableReconnect: filters => {
+    enableReconnect: true,
-      return filters.map(f => ({ ...f, since: newSince }))
-    },
   })
   const relay = await pool.ensureRelay(mockRelay.url)
   relay.pingTimeout = 50
@@ -364,7 +361,7 @@ test('reconnect with filter update in pool', async () => {
   expect(closes).toBe(1)
 
   // check if filter was updated
-  expect(sub.filters[0].since).toBe(newSince)
+  expect(sub.filters[0].since).toBeGreaterThan(1)
 })
 
 test('track relays when publishing', async () => {

relay.test.ts

@@ -336,66 +336,3 @@ test('reconnect on disconnect', async () => {
   expect(relay.connected).toBeTrue()
   expect(closes).toBe(1) // should not have closed again
 })
-
-test('reconnect with filter update', async () => {
-  const mockRelay = new MockRelay()
-  const newSince = Math.floor(Date.now() / 1000)
-  const relay = new Relay(mockRelay.url, {
-    enablePing: true,
-    enableReconnect: filters => {
-      return filters.map(f => ({ ...f, since: newSince }))
-    },
-  })
-  relay.pingTimeout = 50
-  relay.pingFrequency = 50
-  relay.resubscribeBackoff = [50, 100]
-
-  let closes = 0
-  relay.onclose = () => {
-    closes++
-  }
-
-  await relay.connect()
-  expect(relay.connected).toBeTrue()
-
-  const sub = relay.subscribe([{ kinds: [1], since: 0 }], { onevent: () => {} })
-  expect(sub.filters[0].since).toBe(0)
-
-  // wait for the first ping to succeed
-  await new Promise(resolve => setTimeout(resolve, 75))
-  expect(closes).toBe(0)
-
-  // now make it unresponsive
-  mockRelay.unresponsive = true
-
-  // wait for the second ping to fail, which will trigger a close
-  await new Promise(resolve => {
-    const interval = setInterval(() => {
-      if (closes > 0) {
-        clearInterval(interval)
-        resolve(null)
-      }
-    }, 10)
-  })
-  expect(closes).toBe(1)
-  expect(relay.connected).toBeFalse()
-
-  // now make it responsive again
-  mockRelay.unresponsive = false
-
-  // wait for reconnect
-  await new Promise(resolve => {
-    const interval = setInterval(() => {
-      if (relay.connected) {
-        clearInterval(interval)
-        resolve(null)
-      }
-    }, 10)
-  })
-
-  expect(relay.connected).toBeTrue()
-  expect(closes).toBe(1)
-
-  // check if filter was updated
-  expect(sub.filters[0].since).toBe(newSince)
-})

utils.test.ts

@@ -1,6 +1,12 @@
 import { describe, test, expect } from 'bun:test'
 import { buildEvent } from './test-helpers.ts'
-import { Queue, insertEventIntoAscendingList, insertEventIntoDescendingList, binarySearch } from './utils.ts'
+import {
+  Queue,
+  insertEventIntoAscendingList,
+  insertEventIntoDescendingList,
+  binarySearch,
+  normalizeURL,
+} from './utils.ts'
 
 import type { Event } from './core.ts'
 
@@ -263,3 +269,43 @@ test('binary search', () => {
   expect(binarySearch(['a', 'b', 'd', 'e'], b => ('a' < b ? -1 : 'a' === b ? 0 : 1))).toEqual([0, true])
   expect(binarySearch(['a', 'b', 'd', 'e'], b => ('[' < b ? -1 : '[' === b ? 0 : 1))).toEqual([0, false])
 })
+
+describe('normalizeURL', () => {
+  test('normalizes wss:// URLs', () => {
+    expect(normalizeURL('wss://example.com')).toBe('wss://example.com/')
+    expect(normalizeURL('wss://example.com/')).toBe('wss://example.com/')
+    expect(normalizeURL('wss://example.com//path')).toBe('wss://example.com/path')
+    expect(normalizeURL('wss://example.com:443')).toBe('wss://example.com/')
+  })
+
+  test('normalizes https:// URLs', () => {
+    expect(normalizeURL('https://example.com')).toBe('wss://example.com/')
+    expect(normalizeURL('https://example.com/')).toBe('wss://example.com/')
+    expect(normalizeURL('http://example.com//path')).toBe('ws://example.com/path')
+  })
+
+  test('normalizes ws:// URLs', () => {
+    expect(normalizeURL('ws://example.com')).toBe('ws://example.com/')
+    expect(normalizeURL('ws://example.com/')).toBe('ws://example.com/')
+    expect(normalizeURL('ws://example.com//path')).toBe('ws://example.com/path')
+    expect(normalizeURL('ws://example.com:80')).toBe('ws://example.com/')
+  })
+
+  test('adds wss:// to URLs without scheme', () => {
+    expect(normalizeURL('example.com')).toBe('wss://example.com/')
+    expect(normalizeURL('example.com/')).toBe('wss://example.com/')
+    expect(normalizeURL('example.com//path')).toBe('wss://example.com/path')
+  })
+
+  test('handles query parameters', () => {
+    expect(normalizeURL('wss://example.com?z=1&a=2')).toBe('wss://example.com/?a=2&z=1')
+  })
+
+  test('removes hash', () => {
+    expect(normalizeURL('wss://example.com#hash')).toBe('wss://example.com/')
+  })
+
+  test('throws on invalid URL', () => {
+    expect(() => normalizeURL('http://')).toThrow('Invalid URL: http://')
+  })
+})

utils.ts

@@ -9,6 +9,8 @@ export function normalizeURL(url: string): string {
   try {
     if (url.indexOf('://') === -1) url = 'wss://' + url
     let p = new URL(url)
+    if (p.protocol === 'http:') p.protocol = 'ws:'
+    else if (p.protocol === 'https:') p.protocol = 'wss:'
     p.pathname = p.pathname.replace(/\/+/g, '/')
     if (p.pathname.endsWith('/')) p.pathname = p.pathname.slice(0, -1)
     if ((p.port === '80' && p.protocol === 'ws:') || (p.port === '443' && p.protocol === 'wss:')) p.port = ''