parse multiline json from input on nak event and nak req, use iterators instead of channels for more efficient stdin parsing.

This commit is contained in:
fiatjaf
2025-02-05 09:44:16 -03:00
parent 6c634d8081
commit 60d1292f80
3 changed files with 70 additions and 49 deletions

View File

@@ -154,21 +154,20 @@ example:
doAuth := c.Bool("auth") doAuth := c.Bool("auth")
// then process input and generate events // then process input and generate events:
for stdinEvent := range getStdinLinesOrBlank() {
evt := nostr.Event{
Tags: make(nostr.Tags, 0, 3),
}
kindWasSupplied := false // will reuse this
var evt nostr.Event
// this is called when we have a valid json from stdin
handleEvent := func(stdinEvent string) error {
evt.Content = ""
kindWasSupplied := strings.Contains(stdinEvent, `"kind"`)
mustRehashAndResign := false mustRehashAndResign := false
if stdinEvent != "" { if err := easyjson.Unmarshal([]byte(stdinEvent), &evt); err != nil {
if err := easyjson.Unmarshal([]byte(stdinEvent), &evt); err != nil { return fmt.Errorf("invalid event received from stdin: %s", err)
ctx = lineProcessingError(ctx, "invalid event received from stdin: %s", err)
continue
}
kindWasSupplied = strings.Contains(stdinEvent, `"kind"`)
} }
if kind := c.Uint("kind"); slices.Contains(c.FlagNames(), "kind") { if kind := c.Uint("kind"); slices.Contains(c.FlagNames(), "kind") {
@@ -324,6 +323,14 @@ example:
log(nevent + "\n") log(nevent + "\n")
} }
} }
return nil
}
for stdinEvent := range getJsonsOrBlank() {
if err := handleEvent(stdinEvent); err != nil {
ctx = lineProcessingError(ctx, err.Error())
}
} }
exitIfLineProcessingError(ctx) exitIfLineProcessingError(ctx)

View File

@@ -4,11 +4,13 @@ import (
"bufio" "bufio"
"context" "context"
"fmt" "fmt"
"iter"
"math/rand" "math/rand"
"net/http" "net/http"
"net/textproto" "net/textproto"
"net/url" "net/url"
"os" "os"
"slices"
"strings" "strings"
"time" "time"
@@ -43,59 +45,71 @@ func isPiped() bool {
return stat.Mode()&os.ModeCharDevice == 0 return stat.Mode()&os.ModeCharDevice == 0
} }
func getStdinLinesOrBlank() chan string { func getJsonsOrBlank() iter.Seq[string] {
multi := make(chan string) var curr strings.Builder
if hasStdinLines := writeStdinLinesOrNothing(multi); !hasStdinLines {
single := make(chan string, 1) return func(yield func(string) bool) {
single <- "" for stdinLine := range getStdinLinesOrBlank() {
close(single) // we're look for an event, but it may be in multiple lines, so if json parsing fails
return single // we'll try the next line until we're successful
} else { curr.WriteString(stdinLine)
return multi stdinEvent := curr.String()
var dummy any
if err := json.Unmarshal([]byte(stdinEvent), &dummy); err != nil {
continue
}
if !yield(stdinEvent) {
return
}
curr.Reset()
}
} }
} }
func getStdinLinesOrArguments(args cli.Args) chan string { func getStdinLinesOrBlank() iter.Seq[string] {
return func(yield func(string) bool) {
if hasStdinLines := writeStdinLinesOrNothing(yield); !hasStdinLines {
return
} else {
return
}
}
}
func getStdinLinesOrArguments(args cli.Args) iter.Seq[string] {
return getStdinLinesOrArgumentsFromSlice(args.Slice()) return getStdinLinesOrArgumentsFromSlice(args.Slice())
} }
func getStdinLinesOrArgumentsFromSlice(args []string) chan string { func getStdinLinesOrArgumentsFromSlice(args []string) iter.Seq[string] {
// try the first argument // try the first argument
if len(args) > 0 { if len(args) > 0 {
argsCh := make(chan string, 1) return slices.Values(args)
go func() {
for _, arg := range args {
argsCh <- arg
}
close(argsCh)
}()
return argsCh
} }
// try the stdin // try the stdin
multi := make(chan string) return func(yield func(string) bool) {
if !writeStdinLinesOrNothing(multi) { writeStdinLinesOrNothing(yield)
close(multi)
} }
return multi
} }
func writeStdinLinesOrNothing(ch chan string) (hasStdinLines bool) { func writeStdinLinesOrNothing(yield func(string) bool) (hasStdinLines bool) {
if isPiped() { if isPiped() {
// piped // piped
go func() { scanner := bufio.NewScanner(os.Stdin)
scanner := bufio.NewScanner(os.Stdin) scanner.Buffer(make([]byte, 16*1024*1024), 256*1024*1024)
scanner.Buffer(make([]byte, 16*1024*1024), 256*1024*1024) hasEmittedAtLeastOne := false
hasEmittedAtLeastOne := false for scanner.Scan() {
for scanner.Scan() { if !yield(strings.TrimSpace(scanner.Text())) {
ch <- strings.TrimSpace(scanner.Text()) return
hasEmittedAtLeastOne = true
} }
if !hasEmittedAtLeastOne { hasEmittedAtLeastOne = true
ch <- "" }
} if !hasEmittedAtLeastOne {
close(ch) yield("")
}() }
return true return true
} else { } else {
// not piped // not piped

2
req.go
View File

@@ -107,7 +107,7 @@ example:
}() }()
} }
for stdinFilter := range getStdinLinesOrBlank() { for stdinFilter := range getJsonsOrBlank() {
filter := nostr.Filter{} filter := nostr.Filter{}
if stdinFilter != "" { if stdinFilter != "" {
if err := easyjson.Unmarshal([]byte(stdinFilter), &filter); err != nil { if err := easyjson.Unmarshal([]byte(stdinFilter), &filter); err != nil {