wip

README.md

@@ -173,6 +173,13 @@ listening at [wss://relay.damus.io wss://nos.lol wss://relay.nsecbunker.com]:
 {"kind":1,"id":"f030fccd90c783858dfcee204af94826cf0f1c85d6fc85a0087e9e5172419393","pubkey":"79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798","created_at":1719677535,"tags":[["-"],["e","f59911b561c37c90b01e9e5c2557307380835c83399756f4d62d8167227e420a","wss://relay.whatever.com","root","a9e0f110f636f3191644110c19a33448daf09d7cda9708a769e91b7e91340208"],["p","a9e0f110f636f3191644110c19a33448daf09d7cda9708a769e91b7e91340208","wss://p-relay.com"]],"content":"I know the future","sig":"8b36a74e29df8bc12bed66896820da6940d4d9409721b3ed2e910c838833a178cb45fd5bb1c6eb6adc66ab2808bfac9f6644a2c55a6570bb2ad90f221c9c7551"}
 ```
 
+### download the latest 50000 notes from a relay, regardless of their natural query limits, by paginating requests
+```shell
+~> nak req -k 1 --limit 50000 --paginate --paginate-interval 2s nos.lol > events.jsonl
+~> wc -l events.jsonl
+50000 events.jsonl
+```
+
 ## contributing to this repository
 
 Use NIP-34 to send your patches to `naddr1qqpkucttqy28wumn8ghj7un9d3shjtnwdaehgu3wvfnsz9nhwden5te0wfjkccte9ehx7um5wghxyctwvsq3gamnwvaz7tmjv4kxz7fwv3sk6atn9e5k7q3q80cvv07tjdrrgpa0j7j7tmnyl2yr6yr7l8j4s3evf6u64th6gkwsxpqqqpmej2wctpn`.

go.mod

@@ -10,21 +10,24 @@ require (
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0
 	github.com/fatih/color v1.16.0
 	github.com/fiatjaf/cli/v3 v3.0.0-20240723181502-e7dd498b16ae
+	github.com/fiatjaf/eventstore v0.7.1
+	github.com/fiatjaf/khatru v0.7.5
 	github.com/mailru/easyjson v0.7.7
 	github.com/markusmobius/go-dateparser v1.2.3
-	github.com/nbd-wtf/go-nostr v0.34.2
+	github.com/nbd-wtf/go-nostr v0.34.5
 	github.com/nbd-wtf/nostr-sdk v0.0.5
 	golang.org/x/exp v0.0.0-20240707233637-46b078467d37
 )
 
 require (
+	github.com/andybalholm/brotli v1.0.5 // indirect
 	github.com/btcsuite/btcd/btcutil v1.1.3 // indirect
 	github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 // indirect
 	github.com/chzyer/logex v1.1.10 // indirect
 	github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 // indirect
 	github.com/decred/dcrd/crypto/blake256 v1.0.1 // indirect
 	github.com/elliotchance/pie/v2 v2.7.0 // indirect
-	github.com/fiatjaf/eventstore v0.2.16 // indirect
+	github.com/fasthttp/websocket v1.5.7 // indirect
 	github.com/gobwas/httphead v0.1.0 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect
 	github.com/gobwas/ws v1.4.0 // indirect
@@ -32,16 +35,22 @@ require (
 	github.com/hablullah/go-juliandays v1.0.0 // indirect
 	github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
+	github.com/klauspost/compress v1.17.8 // indirect
 	github.com/magefile/mage v1.14.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/puzpuzpuz/xsync/v3 v3.1.0 // indirect
+	github.com/rs/cors v1.7.0 // indirect
+	github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee // indirect
 	github.com/tetratelabs/wazero v1.2.1 // indirect
 	github.com/tidwall/gjson v1.17.1 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/valyala/fasthttp v1.51.0 // indirect
 	github.com/wasilibs/go-re2 v1.3.0 // indirect
 	golang.org/x/crypto v0.21.0 // indirect
+	golang.org/x/net v0.22.0 // indirect
 	golang.org/x/sys v0.22.0 // indirect
 	golang.org/x/text v0.16.0 // indirect
 )

go.sum

@@ -1,4 +1,6 @@
 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
+github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
+github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
 github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c/go.mod h1:tjmYdS6MLJ5/s0Fj4DbLgSbDHbEqLJrtnHecBFkdz5M=
 github.com/btcsuite/btcd v0.23.0/go.mod h1:0QJIIN1wwIXF/3G/m87gIwGniDMDQqjVn4SZgnFpsYY=
@@ -42,12 +44,16 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3
 github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218=
 github.com/elliotchance/pie/v2 v2.7.0 h1:FqoIKg4uj0G/CrLGuMS9ejnFKa92lxE1dEgBD3pShXg=
 github.com/elliotchance/pie/v2 v2.7.0/go.mod h1:18t0dgGFH006g4eVdDtWfgFZPQEgl10IoEO8YWEq3Og=
+github.com/fasthttp/websocket v1.5.7 h1:0a6o2OfeATvtGgoMKleURhLT6JqWPg7fYfWnH4KHau4=
+github.com/fasthttp/websocket v1.5.7/go.mod h1:bC4fxSono9czeXHQUVKxsC0sNjbm7lPJR04GDFqClfU=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/fiatjaf/cli/v3 v3.0.0-20240723181502-e7dd498b16ae h1:0B/1dU3YECIbPoBIRTQ4c0scZCNz9TVHtQpiODGrTTo=
 github.com/fiatjaf/cli/v3 v3.0.0-20240723181502-e7dd498b16ae/go.mod h1:aAWPO4bixZZxPtOnH6K3q4GbQ0jftUNDW9Oa861IRew=
-github.com/fiatjaf/eventstore v0.2.16 h1:NR64mnyUT5nJR8Sj2AwJTd1Hqs5kKJcCFO21ggUkvWg=
-github.com/fiatjaf/eventstore v0.2.16/go.mod h1:rUc1KhVufVmC+HUOiuPweGAcvG6lEOQCkRCn2Xn5VRA=
+github.com/fiatjaf/eventstore v0.7.1 h1:5f2yvEtYvsvMBNttysmXhSSum5M1qwvPzjEQ/BFue7Q=
+github.com/fiatjaf/eventstore v0.7.1/go.mod h1:ek/yWbanKVG767fK51Q3+6Mvi5oEHYSsdPym40nZexw=
+github.com/fiatjaf/khatru v0.7.5 h1:UFo+cdbqHDn1W4Q4h03y3vzh1BiU+6fLYK48oWU2K34=
+github.com/fiatjaf/khatru v0.7.5/go.mod h1:WVqij7X9Vr9UAMIwafQbKVFKxc42Np37vyficwUr/nQ=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
@@ -80,6 +86,8 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
 github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
+github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
+github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo=
 github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
@@ -91,8 +99,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/nbd-wtf/go-nostr v0.34.2 h1:9b4qZ29DhQf9xEWN8/7zfDD868r1jFbpjrR3c+BHc+E=
-github.com/nbd-wtf/go-nostr v0.34.2/go.mod h1:NZQkxl96ggbO8rvDpVjcsojJqKTPwqhP4i82O7K5DJs=
+github.com/nbd-wtf/go-nostr v0.34.5 h1:vti8WqvGWbVoWAPniaz7li2TpCyC+7ZS62Gmy7ib/z0=
+github.com/nbd-wtf/go-nostr v0.34.5/go.mod h1:NZQkxl96ggbO8rvDpVjcsojJqKTPwqhP4i82O7K5DJs=
 github.com/nbd-wtf/nostr-sdk v0.0.5 h1:rec+FcDizDVO0W25PX0lgYMXvP7zNNOgI3Fu9UCm4BY=
 github.com/nbd-wtf/nostr-sdk v0.0.5/go.mod h1:iJJsikesCGLNFZ9dLqhLPDzdt924EagUmdQxT3w2Lmk=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
@@ -108,6 +116,10 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/puzpuzpuz/xsync/v3 v3.1.0 h1:EewKT7/LNac5SLiEblJeUu8z5eERHrmRLnMQL2d7qX4=
 github.com/puzpuzpuz/xsync/v3 v3.1.0/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA=
+github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
+github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
+github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee h1:8Iv5m6xEo1NR1AvpV+7XmhI4r39LGNzwUL4YpMuL5vk=
+github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee/go.mod h1:qwtSXrKuJh/zsFQ12yEE89xfCrGKK63Rr7ctU/uCo4g=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
@@ -122,6 +134,10 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
 github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA=
+github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g=
 github.com/wasilibs/go-re2 v1.3.0 h1:LFhBNzoStM3wMie6rN2slD1cuYH2CGiHpvNL3UtcsMw=
 github.com/wasilibs/go-re2 v1.3.0/go.mod h1:AafrCXVvGRJJOImMajgJ2M7rVmWyisVK7sFshbxnVrg=
 github.com/wasilibs/nottinygc v0.4.0 h1:h1TJMihMC4neN6Zq+WKpLxgd9xCFMw7O9ETLwY2exJQ=

paginate.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"context"
+	"math"
+	"slices"
+	"time"
+
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func paginateWithPoolAndParams(pool *nostr.SimplePool, interval time.Duration, globalLimit uint64) func(ctx context.Context, urls []string, filters nostr.Filters) chan nostr.IncomingEvent {
+	return func(ctx context.Context, urls []string, filters nostr.Filters) chan nostr.IncomingEvent {
+		// filters will always be just one
+		filter := filters[0]
+
+		nextUntil := nostr.Now()
+		if filter.Until != nil {
+			nextUntil = *filter.Until
+		}
+
+		if globalLimit == 0 {
+			globalLimit = uint64(filter.Limit)
+			if globalLimit == 0 && !filter.LimitZero {
+				globalLimit = math.MaxUint64
+			}
+		}
+		var globalCount uint64 = 0
+		globalCh := make(chan nostr.IncomingEvent)
+
+		repeatedCache := make([]string, 0, 300)
+		nextRepeatedCache := make([]string, 0, 300)
+
+		go func() {
+			defer close(globalCh)
+
+			for {
+				filter.Until = &nextUntil
+				time.Sleep(interval)
+
+				keepGoing := false
+				for evt := range pool.SubManyEose(ctx, urls, nostr.Filters{filter}) {
+					if slices.Contains(repeatedCache, evt.ID) {
+						continue
+					}
+
+					keepGoing = true // if we get one that isn't repeated, then keep trying to get more
+					nextRepeatedCache = append(nextRepeatedCache, evt.ID)
+
+					globalCh <- evt
+
+					globalCount++
+					if globalCount >= globalLimit {
+						return
+					}
+
+					if evt.CreatedAt < *filter.Until {
+						nextUntil = evt.CreatedAt
+					}
+				}
+
+				if !keepGoing {
+					return
+				}
+
+				repeatedCache = nextRepeatedCache
+				nextRepeatedCache = nextRepeatedCache[:0]
+			}
+		}()
+
+		return globalCh
+	}
+}

req.go

@@ -96,6 +96,20 @@ example:
 			Usage:       "keep the subscription open, printing all events as they are returned",
 			DefaultText: "false, will close on EOSE",
 		},
+		&cli.BoolFlag{
+			Name:        "paginate",
+			Usage:       "make multiple REQs to the relay decreasing the value of 'until' until 'limit' or 'since' conditions are met",
+			DefaultText: "false",
+		},
+		&cli.DurationFlag{
+			Name:  "paginate-interval",
+			Usage: "time between queries when using --paginate",
+		},
+		&cli.UintFlag{
+			Name:        "paginate-global-limit",
+			Usage:       "global limit at which --paginate should stop",
+			DefaultText: "uses the value given by --limit/-l or infinite",
+		},
 		&cli.BoolFlag{
 			Name:  "bare",
 			Usage: "when printing the filter, print just the filter, not enveloped in a [\"REQ\", ...] array",
@@ -247,7 +261,9 @@ example:
 
 			if len(relayUrls) > 0 {
 				fn := pool.SubManyEose
-				if c.Bool("stream") {
+				if c.Bool("paginate") {
+					fn = paginateWithPoolAndParams(pool, c.Duration("paginate-interval"), c.Uint("paginate-global-limit"))
+				} else if c.Bool("stream") {
 					fn = pool.SubMany
 				}
 

serve.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/fatih/color"
+	"github.com/fiatjaf/cli/v3"
+	"github.com/fiatjaf/eventstore/slicestore"
+	"github.com/fiatjaf/khatru"
+)
+
+var serve = &cli.Command{
+	Name:                      "serve",
+	Usage:                     "starts an in-memory relay for testing purposes",
+	DisableSliceFlagSeparator: true,
+	Flags: []cli.Flag{
+		&cli.UintFlag{
+			Name:  "port",
+			Usage: "port where to listen for connections",
+		},
+		&cli.StringFlag{
+			Name:        "events",
+			Usage:       "file containing the initial batch of events that will be served by the relay as newline-separated JSON (jsonl)",
+			DefaultText: "the relay will start empty",
+		},
+	},
+	Action: func(ctx context.Context, c *cli.Command) error {
+		db := slicestore.SliceStore{}
+
+        reader :=
+		if path := c.String("events"); path != "" {
+			f, err := os.Open(path)
+			if err != nil {
+				return fmt.Errorf("failed to file at '%s': %w", path, err)
+			}
+		} else if isPiped() {
+		}
+
+		rl := khatru.NewRelay()
+		rl.QueryEvents = append(rl.QueryEvents, db.QueryEvents)
+		rl.CountEvents = append(rl.CountEvents, db.CountEvents)
+		rl.DeleteEvent = append(rl.DeleteEvent, db.DeleteEvent)
+		rl.StoreEvent = append(rl.StoreEvent, db.SaveEvent)
+
+		started := make(chan bool)
+		exited := make(chan bool)
+
+		go func() {
+			err := rl.Start("127.0.0.1", int(c.Uint("port")), started)
+			if err != nil {
+				log("error: %s", err)
+			}
+			exited <- true
+		}()
+
+		bold := color.New(color.Bold).Sprint
+		italic := color.New(color.Italic).Sprint
+
+		<-started
+		log()
+	},
+}