change many things, then it finally works somewhat.

This commit is contained in:
fiatjaf 2020-11-13 18:06:47 -03:00
parent bdeb03aeaf
commit 3e551058d7
17 changed files with 377 additions and 107 deletions

2
go.mod
View File

@ -8,8 +8,10 @@ require (
github.com/gorilla/mux v1.8.0
github.com/jmoiron/sqlx v1.2.0
github.com/kelseyhightower/envconfig v1.4.0
github.com/kr/pretty v0.2.1
github.com/lib/pq v1.8.0
github.com/mattn/go-sqlite3 v1.14.4
github.com/rs/cors v1.7.0
github.com/rs/zerolog v1.20.0
gopkg.in/antage/eventsource.v1 v1.0.0-20150318155416-803f4c5af225
)

7
go.sum
View File

@ -33,6 +33,11 @@ github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlT
github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg=
github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
@ -65,6 +70,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/antage/eventsource.v1 v1.0.0-20150318155416-803f4c5af225 h1:xy+AV3uSExoRQc2qWXeZdbhFGwBFK/AmGlrBZEjbvuQ=
gopkg.in/antage/eventsource.v1 v1.0.0-20150318155416-803f4c5af225/go.mod h1:SiXNRpUllqhl+GIw2V/BtKI7BUlz+uxov9vBFtXHqh8=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=

2
relay/Makefile Normal file
View File

@ -0,0 +1,2 @@
relay: $(shell find . -name "*.go")
go build -ldflags="-s -w" -o ./relay

View File

@ -17,16 +17,16 @@ const (
)
type Event struct {
ID string `db:"id"` // it's the hash of the serialized event
ID string `db:"id" json:"id"` // it's the hash of the serialized event
Pubkey string `db:"pubkey"`
Time uint32 `db:"time"`
Pubkey string `db:"pubkey" json:"pubkey"`
CreatedAt uint32 `db:"created_at" json:"created_at"`
Kind uint8 `db:"kind"`
Kind uint8 `db:"kind" json:"kind"`
Reference string `db:"reference"` // the id of another event, optional
Content string `db:"content"`
Signature string `db:"signature"`
Ref string `db:"ref" json:"ref"` // the id of another event, optional
Content string `db:"content" json:"content"`
Sig string `db:"sig" json:"sig"`
}
// Serialize outputs a byte array that can be hashed/signed to identify/authenticate
@ -53,9 +53,9 @@ func (evt *Event) Serialize() ([]byte, error) {
return nil, err
}
// time
// created_at
var timeb [4]byte
binary.BigEndian.PutUint32(timeb[:], evt.Time)
binary.BigEndian.PutUint32(timeb[:], evt.CreatedAt)
if _, err := b.Write(timeb[:]); err != nil {
return nil, err
}
@ -67,12 +67,12 @@ func (evt *Event) Serialize() ([]byte, error) {
return nil, err
}
// reference
if len(evt.Reference) != 0 && len(evt.Reference) != 64 {
// ref
if len(evt.Ref) != 0 && len(evt.Ref) != 64 {
return nil, errors.New("reference must be either blank or 32 bytes")
}
if evt.Reference != "" {
reference, err := hex.DecodeString(evt.Reference)
if evt.Ref != "" {
reference, err := hex.DecodeString(evt.Ref)
if err != nil {
return nil, errors.New("reference is an invalid hex string")
}
@ -97,7 +97,7 @@ func (evt Event) CheckSignature() (bool, error) {
pubkeyb, _ := hex.DecodeString(evt.Pubkey)
pubkey, _ := btcec.ParsePubKey(pubkeyb, btcec.S256())
bsig, err := hex.DecodeString(evt.Signature)
bsig, err := hex.DecodeString(evt.Sig)
if err != nil {
return false, fmt.Errorf("signature is invalid hex: %w", err)
}

View File

@ -6,8 +6,12 @@ import (
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"net/http"
"strings"
"time"
"gopkg.in/antage/eventsource.v1"
)
type ErrorResponse struct {
@ -34,28 +38,70 @@ func queryUsers(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(found)
}
func fetchUserUpdates(w http.ResponseWriter, r *http.Request) {
func listenUpdates(w http.ResponseWriter, r *http.Request) {
w.Header().Set("content-type", "application/json")
key := r.URL.Query().Get("key")
// will return past items then track changes from these keys:
keys, _ := r.URL.Query()["key"]
es := eventsource.New(
eventsource.DefaultSettings(),
func(r *http.Request) [][]byte {
return [][]byte{
[]byte("X-Accel-Buffering: no"),
[]byte("Cache-Control: no-cache"),
[]byte("Content-Type: text/event-stream"),
[]byte("Connection: keep-alive"),
[]byte("Access-Control-Allow-Origin: *"),
}
},
)
go func() {
time.Sleep(2 * time.Second)
es.SendRetryMessage(3 * time.Second)
}()
go func() {
for {
time.Sleep(25 * time.Second)
es.SendEventMessage("", "keepalive", "")
}
}()
es.ServeHTTP(w, r)
// past events
inkeys := make([]string, 0, len(keys))
for _, key := range keys {
// to prevent sql attack here we will check if these keys are valid 33-byte hex
parsed, err := hex.DecodeString(key)
if err != nil || len(parsed) != 33 {
continue
}
inkeys = append(inkeys, fmt.Sprintf("'%x'", parsed))
}
var lastUpdates []Event
err := db.Select(&lastUpdates, `
SELECT *
FROM event
WHERE pubkey = $1
ORDER BY time DESC
LIMIT 25
`, key)
if err == sql.ErrNoRows {
lastUpdates = make([]Event, 0)
} else if err != nil {
WHERE pubkey IN (`+strings.Join(inkeys, ",")+`)
AND created_at > $1
ORDER BY created_at DESC
`, time.Now().AddDate(0, 0, -5).Unix())
if err != nil && err != sql.ErrNoRows {
w.WriteHeader(500)
log.Warn().Err(err).Str("key", key).Msg("failed to fetch updates")
log.Warn().Err(err).Interface("keys", keys).Msg("failed to fetch updates")
return
}
json.NewEncoder(w).Encode(lastUpdates)
for _, evt := range lastUpdates {
jevent, _ := json.Marshal(evt)
es.SendEventMessage(string(jevent), "event", "")
}
// listen to new events
}
func saveUpdate(w http.ResponseWriter, r *http.Request) {
@ -65,19 +111,24 @@ func saveUpdate(w http.ResponseWriter, r *http.Request) {
err := json.NewDecoder(r.Body).Decode(&evt)
if err != nil {
w.WriteHeader(400)
log.Warn().Err(err).Msg("couldn't decode body")
return
}
// safety checks
now := time.Now().UTC().Unix()
if uint32(now-3600) > evt.Time || uint32(now+3600) < evt.Time {
if uint32(now-3600) > evt.CreatedAt || uint32(now+3600) < evt.CreatedAt {
w.WriteHeader(400)
log.Warn().Err(err).Time("now", time.Unix(now, 0)).
Time("event", time.Unix(int64(evt.CreatedAt), 0)).
Msg("time mismatch")
return
}
// check serialization
serialized, err := evt.Serialize()
if err != nil {
log.Warn().Err(err).Msg("serialization error")
w.WriteHeader(400)
return
}
@ -88,10 +139,12 @@ func saveUpdate(w http.ResponseWriter, r *http.Request) {
// check signature (requires the ID to be set)
if ok, err := evt.CheckSignature(); err != nil {
log.Warn().Err(err).Msg("signature verification error")
w.WriteHeader(400)
json.NewEncoder(w).Encode(ErrorResponse{err})
return
} else if !ok {
log.Warn().Err(err).Msg("signature invalid")
w.WriteHeader(400)
json.NewEncoder(w).Encode(ErrorResponse{errors.New("invalid signature")})
return
@ -99,12 +152,12 @@ func saveUpdate(w http.ResponseWriter, r *http.Request) {
// insert
_, err = db.Exec(`
INSERT INTO event (id, pubkey, time, kind, reference, content, signature)
INSERT INTO event (id, pubkey, created_at, kind, ref, content, sig)
VALUES ($1, $2, $3, $4, $5, $6, $7)
`, evt.ID, evt.Pubkey, evt.Time, evt.Kind, evt.Reference, evt.Content, evt.Signature)
`, evt.ID, evt.Pubkey, evt.CreatedAt, evt.Kind, evt.Ref, evt.Content, evt.Sig)
if err != nil {
w.WriteHeader(500)
log.Warn().Err(err).Str("pubkey", evt.Pubkey).Msg("failed to save")
w.WriteHeader(500)
return
}

View File

@ -38,7 +38,7 @@ func main() {
}
router.Path("/query_users").Methods("GET").HandlerFunc(queryUsers)
router.Path("/fetch_user_updates").Methods("GET").HandlerFunc(fetchUserUpdates)
router.Path("/listen_updates").Methods("GET").HandlerFunc(listenUpdates)
router.Path("/save_update").Methods("POST").HandlerFunc(saveUpdate)
srv := &http.Server{

6
relay/notice.go Normal file
View File

@ -0,0 +1,6 @@
package main
type Notice struct {
Kind string `json:"kind"`
Message string `json:"message"`
}

View File

@ -19,9 +19,9 @@ CREATE TABLE event (
pubkey text NOT NULL,
created_at integer NOT NULL,
kind integer NOT NULL,
reference text NOT NULL,
ref text NOT NULL,
content text NOT NULL,
signature text NOT NULL
sig text NOT NULL
);
CREATE UNIQUE INDEX ididx ON event (id);

View File

@ -19,9 +19,9 @@ CREATE TABLE event (
pubkey text NOT NULL,
created_at integer NOT NULL,
kind integer NOT NULL,
reference text NOT NULL,
ref text NOT NULL,
content text NOT NULL,
signature text NOT NULL
sig text NOT NULL
);
CREATE UNIQUE INDEX ididx ON event (id);

2
web-client/Makefile Normal file
View File

@ -0,0 +1,2 @@
static/bundle.js: $(shell ls *.js)
./node_modules/.bin/rollup -c rollup.config.js

View File

@ -1,8 +1,9 @@
{
"dependencies": {
"boxcrate": "^2.1.1",
"buffer": "^6.0.1",
"buffer": "^6.0.2",
"dexie": "^3.0.2",
"elliptic": "^6.5.3",
"insort": "^0.4.0",
"sha.js": "^2.4.11",
"vue": "^3.0.2",
"vue-router": "^4.0.0-rc.2",

View File

@ -1,8 +1,14 @@
<template>
<div class="nav">
<router-link to="/setup">Setup</router-link>
<span class="pubkey">{{ $store.getters.pubKeyHex }}</span>
<router-link to="/">Home</router-link>
<span class="pubkey" :title="$store.getters.pubKeyHex"
>{{ $store.getters.pubKeyHex }}</span
>
<form @submit="submitSearch">
<input v-model="search" />
<button style="display: none">Search</button>
</form>
<router-link to="/setup">Setup</router-link>
</div>
<div>
<router-view />
@ -10,13 +16,24 @@
</template>
<script>
export default {}
export default {
data() {
return {search: ''}
},
methods: {
submitSearch(e) {
e.preventDefault()
this.$router.push('/' + this.search)
}
}
}
</script>
<style>
.nav {
display: flex;
align-items: center;
flex-wrap: wrap;
}
.nav > * {
margin: 0 10px;
@ -27,7 +44,4 @@
text-overflow: ellipsis;
overflow: hidden;
}
.pubkey:hover {
width: auto;
}
</style>

View File

@ -1,24 +1,27 @@
<template>
<h1>Home</h1>
<div v-if="$store.state.following.length">
<h2>Following</h2>
<ul>
<li v-for="key in $store.state.following">
<a :href="'#/' + key">{{ key }}</a>
</li>
</ul>
</div>
<form @submit="publishNote">
<legend>Publishing to {{ writeServersList }}</legend>
<label
>Write anything:
<textarea :disabled="publishing" v-model="text"></textarea>
<input :disabled="publishing" v-model="text" />
</label>
<button :disabled="publishing">Publish</button>
</form>
<p>Data providers: {{ readServersList }}</p>
<div v-if="$store.state.loadingNotes">
<p>Loading notes...</p>
</div>
<div v-else-if="$store.state.following.length === 0">
<p>You're not following anyone.</p>
</div>
<div v-else-if="$store.state.notes.length === 0">
<p>Didn't find any data.</p>
<div v-if="$store.state.notes.size === 0">
<p>Didn't find any notes to show.</p>
</div>
<div v-else>
<div v-for="note in $store.state.notes">
<div v-for="note in $store.state.notes.values()">
<Note v-bind="note" :key="note.id" />
</div>
</div>

View File

@ -1,5 +1,19 @@
<template></template>
<template>
<article>
<div>{{ pubkey }}</div>
<div>{{ content }}</div>
<div>{{ created_at}}</div>
</article>
</template>
<script></script>
<script>
export default {
props: ['id', 'content', 'pubkey', 'created_at', 'signature', 'reference']
}
</script>
<style></style>
<style>
article {
margin: 10px 0;
}
</style>

View File

@ -1,5 +1,35 @@
<template></template>
<template>
<div>
<h1>{{ $route.params.key }}</h1>
<div v-if="following">
<button @click="unfollow">Unfollow</button>
</div>
<div v-else>
<button @click="follow">Follow</button>
</div>
</div>
</template>
<script></script>
<script>
export default {
computed: {
following() {
return (
this.$store.state.following.indexOf(this.$route.params.key) !== -1
)
}
},
methods: {
follow(e) {
e.preventDefault()
this.$store.commit('follow', this.$route.params.key)
},
unfollow(e) {
e.preventDefault()
this.$store.commit('unfollow', this.$route.params.key)
}
}
}
</script>
<style></style>

25
web-client/src/helpers.js Normal file
View File

@ -0,0 +1,25 @@
export function verifySignature(evt) {
return true // TODO
}
export function serializeEvent(evt) {
let version = Buffer.alloc(1)
version.writeUInt8(0)
let pubkey = Buffer.from(evt.pubkey, 'hex')
let time = Buffer.alloc(4)
time.writeUInt32BE(evt.created_at)
let kind = Buffer.alloc(1)
kind.writeUInt8(evt.kind)
let reference = Buffer.alloc(0)
if (evt.ref) {
reference = Buffer.from(evt.ref, 'hex')
}
let content = Buffer.from(evt.content)
return Buffer.concat([version, pubkey, time, kind, reference, content])
}

View File

@ -1,44 +1,42 @@
import {createStore, createLogger} from 'vuex'
import BoxCrate from 'boxcrate'
import elliptic from 'elliptic'
import shajs from 'sha.js'
import Dexie from 'dexie'
import {SortedMap} from 'insort'
import {verifySignature, serializeEvent} from './helpers'
const boxcrate = new BoxCrate({
expiredCheckType: 'active',
expiredCheckInterval: 60000
})
const ec = new elliptic.ec('secp256k1')
const db = new Dexie('db')
db.version(1).stores({
settings: 'key', // as in key => value
relays: 'host',
following: 'pubkey',
mynotes: 'id, kind, created_at',
cachedmetadata: 'pubkey',
cachednotes: 'id, pubkey, created_at'
})
export default createStore({
plugins: process.env.NODE_ENV !== 'production' ? [createLogger()] : [],
plugins: (process.env.NODE_ENV !== 'production'
? [createLogger()]
: []
).concat([init, listener]),
state() {
var relays = {
'http://0.0.0.0:7447': 'rw'
}
var key = null
let following = []
relays = boxcrate.storage.getItem('relays') || relays
key = boxcrate.storage.getItem('key') || key
following = boxcrate.storage.getItem('following') || following
// generate key if doesn't exist
if (key) key = ec.keyFromPrivate(key, 'hex')
else {
key = ec.genKeyPair()
boxcrate.storage.setItem('key', key.getPrivate('hex'))
}
return {
relays,
key,
following
relays: {
'https://relay-profiles.bigsun.xyz': 'rw'
},
key: ec.genKeyPair().getPrivate('hex'),
following: [],
notes: new SortedMap([], (a, b) => a.created_at - b.created_at)
}
},
getters: {
privKeyHex: state => state.key.getPrivate('hex'),
pubKeyHex: state => state.key.getPublic(true, 'hex'),
privKeyHex: state => state.key,
pubKeyHex: state =>
ec.keyFromPrivate(state.key, 'hex').getPublic(true, 'hex'),
writeServers: state =>
Object.keys(state.relays).filter(
host => state.relays[host].indexOf('w') !== -1
@ -48,50 +46,163 @@ export default createStore({
host => state.relays[host].indexOf('r') !== -1
)
},
mutations: {},
mutations: {
setInit(state, {relays, key, following, notes}) {
state.relays = relays
state.key = key
state.following = following
state.notes = notes
},
follow(state, key) {
state.following.push(key)
db.following.put({pubkey: key})
},
unfollow(state, key) {
state.following.splice(state.following.indexOf(key), 1)
db.following.delete(key)
},
receivedEvent(state, evt) {
if (!verifySignature(evt)) {
console.log('received event with invalid signature', evt)
return
}
switch (evt.kind) {
case 0: // setMetadata
break
case 1: // textNote
state.notes.set(evt.id, evt)
break
case 2: // recommendServer
break
}
}
},
actions: {
publishNote(store, text) {
text = text.trim()
let evt = {
pubkey: store.getters.pubKeyHex,
time: Math.round(new Date().getTime() / 1000),
created_at: Math.round(new Date().getTime() / 1000),
kind: 1,
content: text
}
let hash = shajs('sha256').update(serializeEvent(evt)).digest()
evt.id = hash.toString('hex')
evt.signature = store.state.key.sign(hash).toDER('hex')
evt.sig = ec
.keyFromPrivate(store.state.key, 'hex')
.sign(hash, {canonical: true})
.toDER('hex')
for (let i = 0; i < store.getters.writeServers.length; i++) {
let host = store.getters.writeServers[i]
window.fetch(host + '/save_update', {
method: 'POST',
headers: {'content-type': 'application/json'},
body: JSON.stringify(evt)
})
window
.fetch(host + '/save_update', {
method: 'POST',
headers: {'content-type': 'application/json'},
body: JSON.stringify(evt)
})
.then(r => {
if (!r.ok) console.log(`failed to publish ${evt} to ${host}`)
})
}
db.mynotes.put(evt)
store.commit('receivedEvent', evt)
}
}
})
function serializeEvent(evt) {
let version = Buffer.alloc(1)
version.writeUInt8(0)
async function init(store) {
let data = await Promise.all([
db.settings.get('key').then(row => {
if (!row) {
// use the key we generated on startup and save it
db.settings.put({key: 'key', value: store.state.key})
return store.state.key
} else {
// use the key that was stored
return row.value
}
}),
db.relays.toArray().then(rls => {
// if blank, use hardcoded values
if (rls.length === 0) {
return store.state.relays
}
let pubkey = Buffer.from(evt.pubkey, 'hex')
var relays = {}
rls.forEach(({host, policy}) => {
relays[host] = policy
})
let time = Buffer.alloc(4)
time.writeUInt32BE(evt.time)
return relays
}),
db.following.toArray().then(r => r.map(({pubkey}) => pubkey)),
db.mynotes
.orderBy('created_at')
.reverse()
.limit(30)
.toArray()
.then(notes => {
return new SortedMap(
notes.map(n => [n.id, n]),
(a, b) => a.created_at - b.created_at
)
})
])
let kind = Buffer.alloc(1)
kind.writeUInt8(kind)
store.commit('setInit', {
key: data[0],
relays: data[1],
following: data[2],
notes: data[3]
})
}
let reference = Buffer.alloc(0)
if (evt.reference) {
reference = Buffer.from(evt.reference, 'hex')
function listener(store) {
var ess = []
store.subscribe(mutation => {
if (
mutation.type === 'setInit' ||
mutation.type === 'changeRelay' ||
mutation.type === 'follow' ||
mutation.type === 'unfollow'
) {
ess.forEach(es => {
es.close()
})
startListening()
}
})
function startListening() {
store.getters.readServers.forEach(listenToRelay)
}
let content = Buffer.from(evt.content)
function listenToRelay(relayURL, i) {
if (store.state.following.length === 0) return
return Buffer.concat([version, pubkey, time, kind, reference, content])
let qs = store.state.following.map(key => `key=${key}`).join('&')
let es = new EventSource(relayURL + '/listen_updates?' + qs)
ess.push(es)
es.onerror = e => {
console.log(`${relayURL}/listen_updates error: ${e.data}`)
ess.splice(i, 1)
}
es.addEventListener('notice', e => {
console.log(e.data)
})
es.addEventListener('event', e => {
let evt = JSON.parse(e.data)
store.commit('receivedEvent', evt)
})
}
}