misskey.go: improve efficiency of timeline streaming
Iris Lightshard nilix@nilfm.cc
PGP Signature
-----BEGIN PGP SIGNATURE----- iHUEABYKAB0WIQT/foVVmI9pK13hPWFohAcXSWbK8wUCZnuMJQAKCRBohAcXSWbK 89ZwAQC5S1/O+zxryY3BR+GgaUDzfs9aBjUF/VOls3FQ+7z9mgD+JzqpMHJP3E3S xNz34h1k3FQecikqbQH004pdXkhQRAU= =ZxDc -----END PGP SIGNATURE-----
1 files changed,
84 insertions(+),
44 deletions(-)
jump to
M
adapter/misskey.go
→
adapter/misskey.go
@@ -5,7 +5,7 @@ "fmt"
. "forge.lightcrystal.systems/lightcrystal/underbbs/models" "github.com/yitsushi/go-misskey" mkm "github.com/yitsushi/go-misskey/models" - notes "github.com/yitsushi/go-misskey/services/notes" + n "github.com/yitsushi/go-misskey/services/notes" tl "github.com/yitsushi/go-misskey/services/notes/timeline" _ "strings" "time"@@ -25,10 +25,6 @@
cache map[string]time.Time stop chan bool - - notes chan mkm.Note - users chan mkm.User - follows chan mkm.FollowStatus } func (self *MisskeyAdapter) Init(settings Settings, data chan SocketData) error {@@ -76,58 +72,102 @@ // in the background, continuously read data from these API endpoints
// if they are newer than the cache, convert them to UnderBBS objects // and send them on the data channel - go func() { - notesService := self.mk.Notes() - timelineService := notesService.Timeline() + go self.poll() - for { - select { - case _, ok := <-self.stop: - if !ok { - return - } - default: + return nil +} - // TODO: we have to actually decode and pass our filter criteria - tlnotes, tlerr := timelineService.Get(tl.GetRequest{ - Limit: 50, +func (self *MisskeyAdapter) poll() { - }) - mentions, merr := notesService.Mentions(notes.MentionsRequest{ - Limit: 50, - }) + var latest *time.Time - if tlerr != nil { - fmt.Println(tlerr.Error()) - } - if merr != nil { - fmt.Println(merr.Error()) - } + notesService := self.mk.Notes() + timelineService := notesService.Timeline() - // check the cache for everything we just collected - // if anything is newer or as of yet not in the cache, add it - // and convert it to a SocketData implementation before sending on data channel - for _, n := range tlnotes { - msg := self.cacheAndConvert(n) - if msg != nil { - self.data <- msg + for { + select { + case _, ok := <-self.stop: + if !ok { + return + } + default: + + // TODO: we have to actually decode and pass our filter criteria + // probe for new notes + + probenote, err := timelineService.Get(tl.GetRequest{ + Limit: 1, + }) + if err == nil && len(probenote) > 0 && self.isNew(probenote[0]) { + if latest == nil { + latest = &probenote[0].CreatedAt + // this is the first fetch of notes, we can just grab them + notes, err := timelineService.Get(tl.GetRequest{ + Limit: 100, + }) + // if latest is nil also get mentions history + mentions, merr := notesService.Mentions(n.MentionsRequest{ + Limit: 100, + }) + if err != nil { + fmt.Println(err.Error()) } - } - for _, n := range mentions { - msg := self.cacheAndConvert(n) - if msg != nil { - self.data <- msg + if merr != nil { + fmt.Println(merr.Error()) + } + + // check the cache for everything we just collected + // if anything is newer or as of yet not in the cache, add it + // and convert it to a SocketData implementation before sending on data channel + for _, n := range notes { + msg := self.cacheAndConvert(n) + if msg != nil { + self.data <- msg + } + } + for _, n := range mentions { + msg := self.cacheAndConvert(n) + if msg != nil { + self.data <- msg + } + } + + } else { + for { + // get notes since latest, until the probe + notes, err := timelineService.Get(tl.GetRequest{ + SinceDate: uint64(latest.Unix()), + UntilDate: uint64(probenote[0].CreatedAt.Unix()), + Limit: 100, + }) + if err != nil { + fmt.Println(err.Error()) + } + for _, n := range notes { + msg := self.cacheAndConvert(n) + if msg == nil { + + latest = &probenote[0].CreatedAt + break + } + } + if *latest == probenote[0].CreatedAt { + break + } } } - - time.Sleep(5*time.Second) } + time.Sleep(5 * time.Second) } - }() + } - return nil +} + +func (self *MisskeyAdapter) isNew(n mkm.Note) bool { + timestamp, exists := self.cache[n.ID] + return !exists || timestamp.Before(n.CreatedAt) } func (self *MisskeyAdapter) cacheAndConvert(n mkm.Note) *Message {