all repos — underbbs @ 557b87b700e1791b6a0282d81eab183decf59968

decentralized social media client

implement batched fetch and fix adding adapter
Iris Lightshard nilix@nilfm.cc
PGP Signature
-----BEGIN PGP SIGNATURE-----

iHUEABYKAB0WIQT/foVVmI9pK13hPWFohAcXSWbK8wUCZq5gUQAKCRBohAcXSWbK
8z7uAQDZEhwLITM/IuHGXNP9Q32kToq9xSzo6DPaaxQtfEdCXwEAoO5pRJhApsBZ
4UX1PEWGc9fh9wGqne9XnrudGZqHqQE=
=RTBV
-----END PGP SIGNATURE-----
commit

557b87b700e1791b6a0282d81eab183decf59968

parent

0106b445b5f13dfd99f7f8f40743639204ae472e

M adapter/adapter.goadapter/adapter.go

@@ -8,7 +8,7 @@ type Adapter interface {

Init(Settings, chan SocketData) error Name() string Subscribe(string) []error - Fetch(string, string) error + Fetch(string, []string) error Do(string) error DefaultSubscriptionFilter() string }
M adapter/mastodon.goadapter/mastodon.go

@@ -90,7 +90,7 @@ // the stopCh will be closed by a subsequent call to subscribe

return nil } -func (self *MastoAdapter) Fetch(etype string, id string) error { +func (self *MastoAdapter) Fetch(etype string, ids []string) error { return nil }

@@ -110,7 +110,7 @@ Adapter: self.nickname,

Id: fmt.Sprintf("%d", status.ID), Uri: status.URI, Type: "message", - Created: status.CreatedAt.UnixMilli(), + Created: status.CreatedAt.UnixMilli(), }, Content: status.Content, Author: status.Account.Acct,
M adapter/misskey.goadapter/misskey.go

@@ -205,7 +205,7 @@ Uri: n.URI,

Protocol: "misskey", Adapter: self.nickname, Type: "message", - Created: n.CreatedAt.UnixMilli(), + Created: n.CreatedAt.UnixMilli(), }, Author: authorId,

@@ -219,11 +219,11 @@ }

for _, f := range n.Files { msg.Attachments = append(msg.Attachments, Attachment{ - Src: f.URL, - ThumbSrc: f.ThumbnailURL, - Size: f.Size, - Desc: f.Comment, - Created: f.CreatedAt.UnixMilli(), + Src: f.URL, + ThumbSrc: f.ThumbnailURL, + Size: f.Size, + Desc: f.Comment, + Created: f.CreatedAt.UnixMilli(), }) } return &msg

@@ -241,11 +241,11 @@ } else {

authorId = fmt.Sprintf("@%s", usr.Username) } - var updated *int64 = nil - if usr.UpdatedAt != nil { - updatedTmp := usr.UpdatedAt.UnixMilli() - updated = &updatedTmp - } + var updated *int64 = nil + if usr.UpdatedAt != nil { + updatedTmp := usr.UpdatedAt.UnixMilli() + updated = &updatedTmp + } author := Author{ Datagram: Datagram{

@@ -265,76 +265,78 @@

return &author } -func (self *MisskeyAdapter) Fetch(etype, id string) error { - switch etype { - case "message": - data, err := self.mk.Notes().Show(id) - if err != nil { - return err - } else { - msg := self.toMessage(data, true) - if msg != nil { - self.data <- msg - } - } - case "children": - data, err := self.mk.Notes().Children(n.ChildrenRequest{ - NoteID: id, - Limit: 100, - }) - if err != nil { - return err - } else { - for _, n := range data { - msg := self.toMessage(n, true) +func (self *MisskeyAdapter) Fetch(etype, ids []string) error { + for _, id := range ids { + switch etype { + case "message": + data, err := self.mk.Notes().Show(id) + if err != nil { + return err + } else { + msg := self.toMessage(data, true) if msg != nil { self.data <- msg } } - } - case "convoy": - data, err := self.mk.Notes().Conversation(n.ConversationRequest{ - NoteID: id, - Limit: 100, - }) - if err != nil { - return err - } else { - for _, n := range data { - msg := self.toMessage(n, true) - if msg != nil { - self.data <- msg + case "children": + data, err := self.mk.Notes().Children(n.ChildrenRequest{ + NoteID: id, + Limit: 100, + }) + if err != nil { + return err + } else { + for _, n := range data { + msg := self.toMessage(n, true) + if msg != nil { + self.data <- msg + } } } - } - case "author": - user := "" - host := "" - idParts := strings.Split(id, "@") - user = idParts[1] - if len(idParts) == 3 { - host = idParts[2] - } + case "convoy": + data, err := self.mk.Notes().Conversation(n.ConversationRequest{ + NoteID: id, + Limit: 100, + }) + if err != nil { + return err + } else { + for _, n := range data { + msg := self.toMessage(n, true) + if msg != nil { + self.data <- msg + } + } + } + case "author": + user := "" + host := "" + idParts := strings.Split(id, "@") + user = idParts[1] + if len(idParts) == 3 { + host = idParts[2] + } - var hostPtr *string = nil - if len(host) > 0 { - hostPtr = &host - } + var hostPtr *string = nil + if len(host) > 0 { + hostPtr = &host + } - // fmt.Printf("attempting user resolution: @%s@%s\n", user, host) - data, err := self.mk.Users().Show(users.ShowRequest{ - Username: &user, - Host: hostPtr, - }) - if err != nil { - return err - } else { - a := self.toAuthor(data) - if a != nil { - self.data <- a + // fmt.Printf("attempting user resolution: @%s@%s\n", user, host) + data, err := self.mk.Users().Show(users.ShowRequest{ + Username: &user, + Host: hostPtr, + }) + if err != nil { + return err + } else { + a := self.toAuthor(data) + if a != nil { + self.data <- a + } } - } + } } return nil }
M adapter/nostr.goadapter/nostr.go

@@ -79,7 +79,7 @@ fmt.Println("subscription operation completed without errors")

return nil } -func (self *NostrAdapter) Fetch(etype, id string) error { +func (self *NostrAdapter) Fetch(etype, ids []string) error { return nil }
M frontend/ts/adapter-element.tsfrontend/ts/adapter-element.ts

@@ -3,6 +3,7 @@

import { Message, Author } from "./message" import { MessageThread } from "./thread" import { AdapterState } from "./adapter" +import { BatchTimer } from "./batch-timer" export class AdapterElement extends HTMLElement { static observedAttributes = [ "data-latest", "data-view", "data-viewing" ]

@@ -12,6 +13,14 @@ private _view: string = "";

private _name: string = "" private _viewing: string = ""; + private _convoyBatchTimer = new BatchTimer((ids: string[])=>{ + let url = `/api/adapters/${this._name}/fetch?entity_type=convoy`; + for (let id of ids) { + url += `&entity_id=${id}`; + } + util.authorizedFetch("GET", url, null) + }); + // TODO: use visibility of the thread to organize into DMs and public threads private _threads: MessageThread[] = []; private _orphans: Message[] = [];

@@ -29,7 +38,6 @@ this.setAttribute("data-view", "index");

} attributeChangedCallback() { - console.log(`${this._name}.attributeChangedCallback: start`); // set the viewing subject if it's changed const viewing = this.getAttribute("data-viewing"); if (this._viewing != viewing && viewing != null) {

@@ -68,13 +76,11 @@ }

// if latest changed, check if it's a message const latest = this.getAttribute("data-latest"); - console.log(`${this._name}.attributeChangedCallback: checking latest(${latest}) vs _latest${this._latest}`); if (latest ?? "" != this._latest) { - console.log("latest changed") this._latest = latest ?? ""; let datastore = AdapterState._instance.data.get(this._name); if (!datastore) { - util.errMsg(this._name + " has no datastore!"); + //util.errMsg(this._name + " has no datastore!"); return; } const latestMsg = datastore.messages.get(this._latest);

@@ -157,8 +163,11 @@ }

} updateIdxView(latest: string, rootId: string) { - const existingThread = this.querySelector(`underbbs-thread-summary[data-msg="${rootId}"]`); + const existingThread = document.querySelector(`underbbs-thread-summary[data-msg='${rootId}']`); const thread = this._threads.find(t=>t.root.data.id == rootId); + console.log(`looking for thread ${rootId}`) + console.log(`- DOM object: ${existingThread}`); + console.log(`- in memory: ${thread}`); if (existingThread && thread) { console.log(`updating thread: ${thread.root.data.id} // ${thread.messageCount} NEW`) existingThread.setAttribute("data-latest", `${thread.latest}`);

@@ -273,11 +282,7 @@ // otherwise we can orphan it and try to fill it in later

if (this._orphans.filter(o=>o.id == msg.id).length == 0) { this._orphans.push(msg); if (msg.replyTo) { - // request the parent's data, which will try to adopt this orphan when it comes in - util.authorizedFetch( - "GET", - `/api/adapters/${this._name}/fetch?entity_type=message&entity_id=${msg.replyTo}`, - null); + this._convoyBatchTimer.queue(k, 2000); } }
M frontend/ts/adapter.tsfrontend/ts/adapter.ts

@@ -4,12 +4,14 @@ public protocol: string;

public directMessages: Map<string, Message>; public messages: Map<string, Message>; public profileCache: Map<string, Author>; + public convoyCache: Map<string, string>; constructor(protocol: string) { this.protocol = protocol; this.messages = new Map<string, Message>(); this.directMessages = new Map<string, Message>(); this.profileCache = new Map<string, Author>(); + this.convoyCache = new Map<string, string>(); } }
A frontend/ts/batch-timer.ts

@@ -0,0 +1,24 @@

+export class BatchTimer { + private _batch: string[]; + private _timer: number; + private _reqFn: (id: string[])=>void; + + constructor(reqFn: (id: string[])=>void) { + this._batch = []; + this._timer = new Date().getTime(); + this._reqFn = reqFn; + } + + public queue(id: string, timeout: number){ + this._timer = new Date().getTime() + timeout; + this._batch.push(id); + setTimeout(this.checkBatch, timeout); + } + + private checkBatch() { + if ((new Date()).getTime() >= this._timer) { + this._reqFn(this._batch); + this._batch = []; + } + } +}
M frontend/ts/settings-element.tsfrontend/ts/settings-element.ts

@@ -125,7 +125,7 @@ settings.adapters.push(adapterdata);

self._adapters.push(adapterdata.nickname); localStorage.setItem("settings", JSON.stringify(settings)); - self.showSettings(self); + self.showSettings(self)(); } } }
M frontend/ts/util.tsfrontend/ts/util.ts

@@ -1,4 +1,5 @@

import { DatagramSocket } from './websocket' +import { BatchTimer } from './batch-timer' function _(key: string, value: any | null | undefined = undefined): any | null { const x = <any>window;
M models/msg.gomodels/msg.go

@@ -11,8 +11,8 @@ Protocol string `json:"protocol"`

Adapter string `json:"adapter"` Type string `json:"type"` Target *string `json:"target,omitempty"` - Created int64 `json:"created"` - Updated *int64 `json:"updated,omitempty"` + Created int64 `json:"created"` + Updated *int64 `json:"updated,omitempty"` } type Message struct {

@@ -36,11 +36,11 @@ Messages []string `json:"messages,omitempty"`

} type Attachment struct { - Src string `json:"src"` - ThumbSrc string `json:"thumbSrc"` - Desc string `json:"desc"` - Created int64 `json:"created"` - Size uint64 `json:"size"` + Src string `json:"src"` + ThumbSrc string `json:"thumbSrc"` + Desc string `json:"desc"` + Created int64 `json:"created"` + Size uint64 `json:"size"` } type SocketData interface {
M server/api.goserver/api.go

@@ -167,7 +167,7 @@ apiParams := req.Context().Value("params").(map[string]string)

queryParams := req.URL.Query() for _, a := range subscribers[s] { if a.Name() == apiParams["adapter_id"] { - err := a.Fetch(queryParams["entity_type"][0], queryParams["entity_id"][0]) + err := a.Fetch(queryParams["entity_type"][0], queryParams["entity_id"]) if err != nil { fmt.Println(err.Error()) w.WriteHeader(http.StatusInternalServerError)