diff --git a/app/srv/package.json b/app/srv/package.json new file mode 100644 index 00000000..bcd4284f --- /dev/null +++ b/app/srv/package.json @@ -0,0 +1,11 @@ +{ + "name": "srv", + "dependencies": { + "@node-rs/argon2": "^1.5.2", + "@paralleldrive/cuid2": "^2.2.2", + "@types/mime-types": "^2.1.2", + "lz-string": "^1.5.0", + "mime-types": "^2.1.35", + "radix3": "^1.1.0" + } +} diff --git a/app/srv/ws/edit/action/diff-local.ts b/app/srv/ws/edit/action/diff-local.ts new file mode 100644 index 00000000..b7a4563b --- /dev/null +++ b/app/srv/ws/edit/action/diff-local.ts @@ -0,0 +1,87 @@ +import { Websocket } from "hyper-express"; +import { decompress } from "lz-string"; +import * as Y from "yjs"; +import { eg } from "../edit-global"; + +export const diffLocal = (ws: Websocket, msg: any) => { + return new Promise((resolve) => { + const diff_local = Uint8Array.from( + decompress(msg.diff_local) + .split(",") + .map((x) => parseInt(x, 10)) + ); + let doc = null as unknown as Y.Doc; + let wss: Set = null as any; + let um: Y.UndoManager = null as any; + if (msg.mode === "page") { + doc = eg.edit.page[msg.id].doc as any; + wss = eg.edit.page[msg.id].ws; + um = eg.edit.page[msg.id].undoManager; + } else if (msg.mode === "comp") { + doc = eg.edit.comp[msg.id].doc as any; + wss = eg.edit.comp[msg.id].ws; + um = eg.edit.comp[msg.id].undoManager; + } else if (msg.mode === "site") { + doc = eg.edit.site[msg.id].doc as any; + wss = eg.edit.site[msg.id].ws; + um = eg.edit.site[msg.id].undoManager; + } + + if (doc && wss) { + Y.applyUpdate(doc, diff_local); + + if (msg.mode === "page") { + clearTimeout(eg.edit.page[msg.id].saveTimeout); + eg.edit.page[msg.id].saveTimeout = setTimeout(async () => { + if (msg.id) { + const page = eg.edit.page[msg.id].doc.getMap("map").toJSON(); + try { + await db.page.update({ + where: { id: page.id }, + data: { + content_tree: page.content_tree, + updated_at: new Date(), + }, + }); + resolve(); + } catch (e) { + console.error(e); + console.error({ + ...page, + updated_at: new Date(), + }); + } + } + }, 1500); + } else if (msg.mode === "comp") { + eg.edit.comp[msg.id].saveTimeout = setTimeout(async () => { + const comp = eg.edit.comp[msg.id].doc.getMap("map").toJSON(); + await db.component.update({ + where: { + id: msg.id, + }, + data: { + name: comp.name, + content_tree: comp.content_tree, + updated_at: new Date(), + }, + }); + }, 1500); + } else if (msg.mode === "site") { + clearTimeout(eg.edit.site[msg.id].saveTimeout); + eg.edit.site[msg.id].saveTimeout = setTimeout(async () => { + const site = eg.edit.site[msg.id].doc.getMap("site").toJSON(); + delete site.page; + await db.site.update({ + where: { + id: msg.id, + }, + data: { + ...site, + }, + }); + }, 1500); + } + } + }); +}; diff --git a/app/srv/ws/edit/action/get-comp.ts b/app/srv/ws/edit/action/get-comp.ts new file mode 100644 index 00000000..44a0b809 --- /dev/null +++ b/app/srv/ws/edit/action/get-comp.ts @@ -0,0 +1,71 @@ +import { Websocket } from "hyper-express"; +import { compress } from "lz-string"; +import { syncronize } from "y-pojo"; +import * as Y from "yjs"; +import { + WS_MSG_GET_COMP, + WS_MSG_SET_COMP, + WS_MSG_SV_LOCAL, +} from "../../../web/src/utils/types/ws"; +import { SingleComp, eg } from "../edit-global"; + +export const getComp = async (ws: Websocket, msg: WS_MSG_GET_COMP) => { + const comp_id = msg.comp_id; + + if (!eg.edit.comp[comp_id]) { + const rawComp = await db.component.findFirst({ + where: { + id: comp_id, + }, + }); + + if (!rawComp) { + const sent: WS_MSG_SET_COMP = { + type: "set_comp", + comp_id: comp_id, + changes: "", + }; + ws.send(JSON.stringify(sent)); + return; + } + + if (rawComp) { + const ydoc = new Y.Doc() as SingleComp["doc"]; + const map = ydoc.getMap("map"); + syncronize(map as any, rawComp); + + const ws = new Set(); + const um = new Y.UndoManager(map, { ignoreRemoteMapChanges: true }); + const broadcast = () => { + const sv_local = compress(Y.encodeStateVector(ydoc as any).toString()); + const broadcast: WS_MSG_SV_LOCAL = { + type: "sv_local", + sv_local, + mode: "comp", + id: comp_id, + }; + ws.forEach((w) => w.send(JSON.stringify(broadcast))); + }; + um.on("stack-item-added", broadcast); + um.on("stack-item-updated", broadcast); + + eg.edit.comp[comp_id] = { + doc: ydoc, + id: comp_id, + undoManager: um, + ws, + }; + } + } + + const comp = eg.edit.comp[comp_id]; + if (comp) { + if (!comp.ws.has(ws)) comp.ws.add(ws); + const sent: WS_MSG_SET_COMP = { + type: "set_comp", + comp_id: comp_id, + changes: compress(Y.encodeStateAsUpdate(comp.doc as any).toString()), + }; + ws.send(JSON.stringify(sent)); + } +}; diff --git a/app/srv/ws/edit/action/get-page.ts b/app/srv/ws/edit/action/get-page.ts new file mode 100644 index 00000000..1e606941 --- /dev/null +++ b/app/srv/ws/edit/action/get-page.ts @@ -0,0 +1,74 @@ +import { Websocket } from "hyper-express"; +import { compress } from "lz-string"; +import { syncronize } from "y-pojo"; +import * as Y from "yjs"; +import { + WS_MSG_GET_PAGE, + WS_MSG_SET_PAGE, + WS_MSG_SV_LOCAL, +} from "../../../web/src/utils/types/ws"; +import { MPage } from "../../../web/src/utils/types/general"; +import { eg } from "../edit-global"; +import { loadPage } from "../tools/load-page"; +import { validate } from "uuid"; + +export const getPage = async (ws: Websocket, msg: WS_MSG_GET_PAGE) => { + const page_id = msg.page_id; + if (!validate(page_id)) return; + if (!eg.edit.page[page_id]) { + const rawPage = await loadPage(page_id); + + if (rawPage) { + const ydoc = new Y.Doc() as MPage; + let root = ydoc.getMap("map"); + syncronize(root as any, rawPage); + const ws = new Set(); + const um = new Y.UndoManager(root, { ignoreRemoteMapChanges: true }); + const broadcast = () => { + const sv_local = compress(Y.encodeStateVector(ydoc as any).toString()); + const broadcast: WS_MSG_SV_LOCAL = { + type: "sv_local", + sv_local, + mode: "page", + id: page_id, + }; + ws.forEach((w) => w.send(JSON.stringify(broadcast))); + }; + um.on("stack-item-added", broadcast); + um.on("stack-item-updated", broadcast); + + eg.edit.page[page_id] = { + id: page_id, + doc: ydoc, + undoManager: um, + ws, + }; + } + } + + const page = eg.edit.page[page_id]; + // let root = page.doc.getMap("map").get("content_tree") as unknown as MContent; + // if (root) { + // let changed = false; + // await page.doc.transact(async () => { + // changed = await validateTreePage(ws, root); + // }); + + // if (changed) { + // root = page.doc.getMap("map").get("content_tree") as unknown as MContent; + // await db.page.update({ + // where: { + // id: page.id, + // }, + // data: { content_tree: root.toJSON(), updated_at: new Date() }, + // }); + // } + // } + + page.ws.add(ws); + const sent: WS_MSG_SET_PAGE = { + type: "set_page", + changes: compress(Y.encodeStateAsUpdate(page.doc as any).toString()), + }; + ws.send(JSON.stringify(sent)); +}; diff --git a/app/srv/ws/edit/action/sv-local.ts b/app/srv/ws/edit/action/sv-local.ts new file mode 100644 index 00000000..1df6a97c --- /dev/null +++ b/app/srv/ws/edit/action/sv-local.ts @@ -0,0 +1,48 @@ +import { Websocket } from "hyper-express"; +import { compress, decompress } from "lz-string"; +import * as Y from "yjs"; +import { + WS_MSG_SVDIFF_REMOTE, + WS_MSG_SV_LOCAL, +} from "../../../web/src/utils/types/ws"; +import { eg } from "../edit-global"; +import { getComp } from "./get-comp"; +import { getPage } from "./get-page"; + +export const svLocal = async (ws: Websocket, msg: WS_MSG_SV_LOCAL) => { + const changes = Uint8Array.from( + decompress(msg.sv_local) + .split(",") + .map((x) => parseInt(x, 10)) + ); + let doc = null as any; + if (msg.mode === "page") { + if (!eg.edit.page[msg.id]) { + await getPage(ws, { type: "get_page", page_id: msg.id }); + } + + doc = eg.edit.page[msg.id].doc; + } else if (msg.mode === "comp") { + if (!eg.edit.comp[msg.id]) { + await getComp(ws, { comp_id: msg.id, type: "get_comp" }); + } + + doc = eg.edit.comp[msg.id].doc; + } else if (msg.mode === "site") { + doc = eg.edit.site[msg.id].doc; + } + + if (doc) { + const diff_remote = Y.encodeStateAsUpdate(doc, changes); + const sv_remote = Y.encodeStateVector(doc); + + const sendmsg: WS_MSG_SVDIFF_REMOTE = { + diff_remote: compress(diff_remote.toString()), + sv_remote: compress(sv_remote.toString()), + id: msg.id, + mode: msg.mode, + type: "svd_remote", + }; + ws.send(JSON.stringify(sendmsg)); + } +}; diff --git a/app/srv/ws/edit/action/svdiff-remote.ts b/app/srv/ws/edit/action/svdiff-remote.ts new file mode 100644 index 00000000..e83d5c01 --- /dev/null +++ b/app/srv/ws/edit/action/svdiff-remote.ts @@ -0,0 +1,46 @@ +import { Websocket } from "hyper-express"; +import { compress, decompress } from "lz-string"; +import * as Y from "yjs"; +import { + WS_MSG_DIFF_LOCAL, + WS_MSG_SVDIFF_REMOTE, +} from "../../../web/src/utils/types/ws"; +import { eg } from "../edit-global"; + +export const svdiffRemote = async ( + ws: Websocket, + msg: WS_MSG_SVDIFF_REMOTE +) => { + const sv_remote = Uint8Array.from( + decompress(msg.sv_remote) + .split(",") + .map((x) => parseInt(x, 10)) + ); + const diff_remote = Uint8Array.from( + decompress(msg.diff_remote) + .split(",") + .map((x) => parseInt(x, 10)) + ); + + let doc = null as any; + if (msg.mode === "page") { + doc = eg.edit.page[msg.id].doc; + } else if (msg.mode === "comp") { + doc = eg.edit.comp[msg.id].doc; + } else if (msg.mode === "site") { + doc = eg.edit.site[msg.id].doc; + } + + if (doc) { + const diff_local = Y.encodeStateAsUpdate(doc as any, sv_remote); + Y.applyUpdate(doc as any, diff_remote); + + const sendmsg: WS_MSG_DIFF_LOCAL = { + type: "diff_local", + mode: msg.mode, + id: msg.id, + diff_local: compress(diff_local.toString()), + }; + ws.send(JSON.stringify(sendmsg)); + } +}; diff --git a/app/srv/ws/edit/action/undo-redo.ts b/app/srv/ws/edit/action/undo-redo.ts new file mode 100644 index 00000000..7e18283a --- /dev/null +++ b/app/srv/ws/edit/action/undo-redo.ts @@ -0,0 +1,37 @@ +import { Websocket } from "hyper-express"; +import { eg } from "../edit-global"; +import { UndoManager } from "yjs"; +import { WS_MSG_REDO, WS_MSG_UNDO } from "../../../web/src/utils/types/ws"; + +export const undo = (ws: Websocket, msg: WS_MSG_UNDO) => { + const um = getUndoManager(msg); + if (um && um.canUndo()) { + um.undo(); + } +}; + +export const redo = (ws: Websocket, msg: WS_MSG_REDO) => { + const um = getUndoManager(msg); + if (um && um.canRedo()) { + um.redo(); + } +}; + +const getUndoManager = (msg: WS_MSG_UNDO | WS_MSG_REDO) => { + let undoManager = null as null | UndoManager; + if (msg.mode === "page") { + if (eg.edit.page[msg.id]) { + undoManager = eg.edit.page[msg.id].undoManager; + } + } else if (msg.mode === "site") { + if (eg.edit.site[msg.id]) { + undoManager = eg.edit.site[msg.id].undoManager; + } + } else if (msg.mode === "comp") { + if (eg.edit.comp[msg.id]) { + undoManager = eg.edit.comp[msg.id].undoManager; + } + } + + return undoManager; +}; diff --git a/app/srv/ws/edit/edit-global.ts b/app/srv/ws/edit/edit-global.ts new file mode 100644 index 00000000..0d5c06ef --- /dev/null +++ b/app/srv/ws/edit/edit-global.ts @@ -0,0 +1,70 @@ +import { ServerWebSocket } from "bun"; +import { component } from "dbgen"; +import { UndoManager } from "yjs"; +import { TypedArray, TypedDoc, TypedMap } from "yjs-types"; +import type { WSData } from "../../../../pkgs/core/server/create"; + +import { IItem } from "../../../web/src/utils/types/item"; +import { IRoot } from "../../../web/src/utils/types/root"; +import { Site } from "./tools/load-site"; +import { MPage } from "../../../web/src/utils/types/general"; +import type { RadixRouter } from "radix3"; + +type ArrayElement = + ArrayType extends readonly (infer ElementType)[] ? ElementType : never; + +export type SingleComp = { + id: string; + doc: TypedDoc<{ + map: TypedMap }>; + }>; + undoManager: UndoManager; + saveTimeout?: ReturnType; + ws: Set>; +}; + +export const eg = global as unknown as { + cache: Record< + string, + Record< + string, + { + id: string; + js: string | null; + url: string; + js_compiled: string | null; + content_tree: IRoot; + lastRefresh: number; + } + > + >; + router: Record>; + edit: { + site: Record< + string, + { + id: string; + doc: TypedDoc<{ + site: TypedMap< + Site & { page: TypedArray> } + >; + }>; + undoManager: UndoManager; + saveTimeout?: ReturnType; + ws: Set>; + } + >; + comp: Record; + page: Record< + string, + { + id: string; + doc: MPage; + undoManager: UndoManager; + saveTimeout?: ReturnType; + ws: Set>; + } + >; + ws: WeakMap, { clientID: string }>; + }; +}; diff --git a/app/srv/ws/edit/tools/load-page-cached.ts b/app/srv/ws/edit/tools/load-page-cached.ts new file mode 100644 index 00000000..af09e96e --- /dev/null +++ b/app/srv/ws/edit/tools/load-page-cached.ts @@ -0,0 +1,47 @@ +import { eg } from "../edit-global"; + +// page cache timeout, in seconds +const PAGE_CACHE_TIMEOUT = 1; + +export const loadCachedPage = async (site_id: string, page_id: string) => { + let site_cache = eg.cache[site_id]; + if (!site_cache) { + eg.cache[site_id] = {}; + site_cache = eg.cache[site_id]; + } + + if (!page_id) return {}; + + let cache = site_cache[page_id]; + if ( + !cache || + (cache && !cache.lastRefresh) || + (cache && Date.now() - cache.lastRefresh >= 1000 * PAGE_CACHE_TIMEOUT) + ) { + if (eg.edit.page[page_id]) { + const edit = eg.edit.page[page_id].doc.getMap("map").toJSON(); + edit.lastRefresh = Date.now(); + site_cache[page_id] = edit as any; + } else { + const page = await db.page.findFirst({ + where: { id: page_id }, + select: { + js: true, + id: true, + url: true, + updated_at: true, + js_compiled: true, + content_tree: true, + }, + }); + if (page) { + site_cache[page_id] = { + ...page, + lastRefresh: Date.now(), + } as any; + } + } + cache = site_cache[page_id]; + } + return cache; +}; diff --git a/app/srv/ws/edit/tools/load-page.ts b/app/srv/ws/edit/tools/load-page.ts new file mode 100644 index 00000000..201f3501 --- /dev/null +++ b/app/srv/ws/edit/tools/load-page.ts @@ -0,0 +1,22 @@ +import { validate } from "uuid"; +import { Page } from "../../../../web/src/utils/types/general"; + +export const loadPage = async (page_id: string) => { + if (page_id && validate(page_id)) { + let page = (await db.page.findFirst({ + where: { id: page_id }, + select: { + id: true, + js: true, + name: true, + id_site: true, + url: true, + js_compiled: true, + updated_at: true, + content_tree: true, + }, + })) as unknown as null | Page; + return page; + } + return null; +}; diff --git a/app/srv/ws/edit/tools/load-site.ts b/app/srv/ws/edit/tools/load-site.ts new file mode 100644 index 00000000..bf2909c2 --- /dev/null +++ b/app/srv/ws/edit/tools/load-site.ts @@ -0,0 +1,95 @@ +import { page, site } from "dbgen"; +import { validate as isValidUUID } from "uuid"; +import * as Y from "yjs"; + +export type SiteConfig = { + api_url?: string; + prasi?: { + port: number; + dburl: string; + }; +}; +export type Site = Exclude>, null>; + +export const loadSite = async (idOrDomain: string) => { + let rname = idOrDomain; + if (!rname) { + rname = "prasi.app"; + } + + const res = await db.site.findFirst({ + where: isValidUUID(rname) + ? { + id: rname, + } + : { + domain: rname, + }, + include: { + page: { + select: { + id: true, + url: true, + updated_at: true, + name: true, + }, + }, + }, + }); + + if (res) { + if (!res.page) { + res.page = []; + } + if (res.page.length === 0) { + const page = await createPage(res as any, { + name: "Home", + url: "/", + }); + + res.page.push(page); + } + + for (const p of res.page) { + const page = p as any; + page.js = new Y.Text(); + page.js_compiled = new Y.Text(); + page.content_tree = new Y.Map(); + } + } + + return res as + | (Omit & { + config?: SiteConfig; + page: { + id: string; + url: string; + updated_at: Date | null; + name: string; + }[]; + }) + | null; +}; + +const createPage = async ( + site: site & { page: page[] }, + page: WithOptional< + Parameters[0]["data"], + "content_tree" + > +) => { + const raw = await db.page.create({ + data: { + ...(page as any), + content_tree: page.content_tree ? page.content_tree : blank, + site: { + connect: { id: site.id }, + }, + }, + }); + + return raw; +}; + +const blank = { id: "root", type: "root", childs: [] }; +type WithOptional = Omit & Partial>; diff --git a/app/srv/ws/edit/tools/validate-tree-page.ts b/app/srv/ws/edit/tools/validate-tree-page.ts new file mode 100644 index 00000000..ad2f7e1f --- /dev/null +++ b/app/srv/ws/edit/tools/validate-tree-page.ts @@ -0,0 +1,54 @@ +import { Websocket } from "hyper-express"; +import { syncronize } from "y-pojo"; +import * as Y from "yjs"; +import { fillID } from "../../../web/src/utils/page/tools/fill-id"; +import { IContent, MContent } from "../../../web/src/utils/types/general"; +import { IItem } from "../../../web/src/utils/types/item"; +import { IRoot } from "../../../web/src/utils/types/root"; +import { getComp } from "../action/get-comp"; +import { eg } from "../edit-global"; + +const MAX_STRING_LENGTH = 15000; +export const validateTreeMap = async ( + ws: Websocket, + item: MContent, + changed?: boolean +) => { + let _changed = changed; + const type = item.get("type") as IContent["type"] | "root"; + if (type !== "root") { + item.forEach((val, key, map) => { + if (typeof val === "string") { + if (val.length > MAX_STRING_LENGTH) { + map.set(key, ""); + _changed = true; + } + } else { + if (typeof val === "object" && val instanceof Y.Map) { + val._map.forEach((ival, ikey, imap) => { + if (typeof ival === "string") { + if ((ival as string).length > MAX_STRING_LENGTH) { + imap.set(ikey, "" as any); + _changed = true; + } + } + }); + } + } + }); + } + + if (item) { + if (type !== "text") { + const childs = item.get("childs"); + if (childs) { + for (const c of childs) { + if (await validateTreeMap(ws, c)) { + _changed = true; + } + } + } + } + } + return !!_changed; +}; diff --git a/app/srv/ws/handler.ts b/app/srv/ws/handler.ts index 47b6ce36..d0bcf87e 100644 --- a/app/srv/ws/handler.ts +++ b/app/srv/ws/handler.ts @@ -1,10 +1,101 @@ +import { createId } from "@paralleldrive/cuid2"; import { WebSocketHandler } from "bun"; +import { WSData } from "../../../pkgs/core/server/create"; +import { WS_MSG } from "../../web/src/utils/types/ws"; +import { eg } from "./edit/edit-global"; +import { decompress } from "lz-string"; +import { getPage } from "./edit/action/get-page"; +import { getComp } from "./edit/action/get-comp"; +import { svLocal } from "./edit/action/sv-local"; +import { diffLocal } from "./edit/action/diff-local"; +import { svdiffRemote } from "./edit/action/svdiff-remote"; +import { redo, undo } from "./edit/action/undo-redo"; -export const wsHandler: Record> = { +eg.edit = { + site: {}, + comp: {}, + page: {}, + ws: new WeakMap(), +}; +const site = { + saveTimeout: null as any, +}; + +export const wsHandler: Record> = { "/edit": { - open(ws) {}, - message(ws, message) {}, - close(ws, code, reason) {}, + open(ws) { + eg.edit.ws.set(ws, { + clientID: createId(), + }); + }, + async message(ws, raw) { + if (typeof raw === "string") { + try { + const msg = JSON.parse(raw) as WS_MSG; + + if (msg.type === "ping") { + ws.send(JSON.stringify({ type: "pong" })); + return; + } + + switch (msg.type) { + case "site-js": + clearTimeout(site.saveTimeout); + site.saveTimeout = setTimeout(async () => { + const js = JSON.parse(decompress(msg.src)); + await db.site.update({ + where: { + id: msg.id_site, + }, + data: { + js: js.src, + js_compiled: js.compiled, + }, + }); + }, 1000); + break; + case "get_page": + await getPage(ws, msg); + break; + case "get_comp": + await getComp(ws, msg); + break; + case "sv_local": + await svLocal(ws, msg); + break; + case "diff_local": + await diffLocal(ws, msg); + break; + case "svd_remote": + await svdiffRemote(ws, msg); + break; + case "undo": + undo(ws, msg); + break; + case "redo": + redo(ws, msg); + break; + } + } catch (e) { + console.log(e); + } + } + }, + close(ws, code, reason) { + eg.edit.ws.delete(ws); + + for (const page of Object.values(eg.edit.page)) { + page.ws.delete(ws); + } + + for (const site of Object.values(eg.edit.site)) { + site.ws.delete(ws); + } + + for (const comp of Object.values(eg.edit.comp)) { + comp.ws.delete(ws); + } + }, drain(ws) {}, }, }; diff --git a/bun.lockb b/bun.lockb index 46f913d2..72c44113 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/package.json b/package.json index 4348a791..ba259d6f 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,5 @@ }, "peerDependencies": { "typescript": "^5.0.0" - }, - "dependencies": { "@node-rs/argon2": "^1.5.2", "@types/mime-types": "^2.1.2", "mime-types": "^2.1.35" } + } } diff --git a/pkgs/core/server/create.ts b/pkgs/core/server/create.ts index 5652ce7e..a14e7c38 100644 --- a/pkgs/core/server/create.ts +++ b/pkgs/core/server/create.ts @@ -7,12 +7,15 @@ import { WebSocketHandler } from "bun"; const cache = { static: {} as Record }; +export type WSData = { url: URL }; + export const createServer = async () => { g.api = {}; g.router = createRouter({ strictTrailingSlash: true }); g.server = Bun.serve({ port: g.port, websocket: { + maxPayloadLength: 99999999, close(ws, code, reason) { const pathname = ws.data.url.pathname; if (wsHandler[pathname]) { @@ -40,7 +43,7 @@ export const createServer = async () => { } } }, - } as WebSocketHandler<{ url: URL }>, + } as WebSocketHandler, async fetch(req, server) { if (g.status === "init") return new Response("initializing..."); const url = new URL(req.url);