diff --git a/app/srv/package.json b/app/srv/package.json index d34a8386..1ea4ea1b 100644 --- a/app/srv/package.json +++ b/app/srv/package.json @@ -11,6 +11,9 @@ "lmdb": "^2.8.5", "mime-types": "^2.1.35", "msgpackr": "^1.9.9", - "radix3": "^1.1.0" + "radix3": "^1.1.0", + "y-pojo": "^0.0.8", + "yjs": "^13.6.8", + "yjs-types": "^0.0.1" } } diff --git a/app/srv/ws/edit/action/sv-local.ts b/app/srv/ws/edit/action/sv-local.ts index 761074c2..219cfac2 100644 --- a/app/srv/ws/edit/action/sv-local.ts +++ b/app/srv/ws/edit/action/sv-local.ts @@ -32,7 +32,6 @@ export const svLocal = async (ws: any, msg: WS_MSG_SV_LOCAL) => { if (doc) { const diff_remote = Y.encodeStateAsUpdate(doc, changes); const sv_remote = Y.encodeStateVector(doc); - const sendmsg: WS_MSG_SVDIFF_REMOTE = { diff_remote: diff_remote.toString(), sv_remote: sv_remote.toString(), diff --git a/app/srv/ws/sync/actions.ts b/app/srv/ws/sync/actions.ts index 234e830e..5a645c7f 100644 --- a/app/srv/ws/sync/actions.ts +++ b/app/srv/ws/sync/actions.ts @@ -1,5 +1,5 @@ -import { component, site, page } from "dbgen"; -import { ESite } from "../../../web/src/render/ed/logic/ed-global"; +import { component, page } from "dbgen"; +import { EPage, ESite } from "../../../web/src/render/ed/logic/ed-global"; /* WARNING: @@ -15,7 +15,7 @@ export const SyncActions = { list: async () => ({}) as Record, group: async () => ({}) as Record, - load: async (id: string) => ({}) as ESite | undefined, + load: async (id: string) => ({}) as ESite | void, }, comp: { list: () => ({}) as Record>, @@ -25,6 +25,6 @@ export const SyncActions = { page: { list: (id_site: string) => ({}) as Record>, - load: (id: string) => ({}) as Uint8Array, + load: async (id: string) => ({}) as EPage | void, }, }; diff --git a/app/srv/ws/sync/actions/index.ts b/app/srv/ws/sync/actions/index.ts index cf2dd000..77660dbc 100644 --- a/app/srv/ws/sync/actions/index.ts +++ b/app/srv/ws/sync/actions/index.ts @@ -1,2 +1,3 @@ export * from "./site_load"; export * from "./site_group"; +export * from "./page_load"; diff --git a/app/srv/ws/sync/actions/page_load.ts b/app/srv/ws/sync/actions/page_load.ts new file mode 100644 index 00000000..089bf093 --- /dev/null +++ b/app/srv/ws/sync/actions/page_load.ts @@ -0,0 +1,55 @@ +import { syncronize } from "y-pojo"; +import { SAction } from "../actions"; +import { Y, docs } from "../entity/docs"; +import { snapshot } from "../entity/snapshot"; +import { ActionCtx } from "../type"; + +export const page_load: SAction["page"]["load"] = async function ( + this: ActionCtx, + id: string +) { + let ss = snapshot.get("page", id); + let ydoc = docs.page[id]; + + if (!ss || !ydoc) { + const page = await db.page.findFirst({ where: { id } }); + if (page) { + const doc = new Y.Doc(); + let root = doc.getMap("map"); + syncronize(root, { id, root: page.content_tree }); + + const um = new Y.UndoManager(root, { ignoreRemoteMapChanges: true }); + docs.page[id] = { + doc: doc as any, + id, + um, + }; + + const bin = Y.encodeStateAsUpdate(doc); + snapshot.set({ + bin, + id, + type: "page", + name: page.name, + ts: Date.now(), + url: page.url, + }); + + return { + id: id, + url: page.url, + name: page.name, + snapshot: bin, + }; + } + } + + if (ss) { + return { + id: ss.id, + url: ss.url, + name: ss.name, + snapshot: ss.bin, + }; + } +}; diff --git a/app/srv/ws/sync/editor/load.ts b/app/srv/ws/sync/editor/load.ts index 00f7c057..0b2761af 100644 --- a/app/srv/ws/sync/editor/load.ts +++ b/app/srv/ws/sync/editor/load.ts @@ -1,4 +1,4 @@ -import { user } from "../user"; +import { user } from "../entity/user"; export const loadDefaultSite = async (user_id: string) => { const conf = user.conf.get(user_id); diff --git a/app/srv/ws/sync/entity/docs.ts b/app/srv/ws/sync/entity/docs.ts new file mode 100644 index 00000000..0b5f288b --- /dev/null +++ b/app/srv/ws/sync/entity/docs.ts @@ -0,0 +1,24 @@ +import * as Y from "yjs"; +import { TypedDoc, TypedMap } from "yjs-types"; +import { MItem } from "../../../../web/src/utils/types/item"; +import { DPage } from "../../../../web/src/utils/types/root"; +export * as Y from "yjs"; + +export const docs = { + page: {} as Record< + string, + { + id: string; + doc: DPage; + um: Y.UndoManager; + } + >, + comp: {} as Record< + string, + { + id: string; + doc: TypedDoc<{ map: TypedMap<{ id: string; item: MItem }> }>; + um: Y.UndoManager; + } + >, +}; diff --git a/app/srv/ws/sync/entity/snapshot.ts b/app/srv/ws/sync/entity/snapshot.ts new file mode 100644 index 00000000..89200fbf --- /dev/null +++ b/app/srv/ws/sync/entity/snapshot.ts @@ -0,0 +1,46 @@ +import { dir } from "dir"; +import { RootDatabase, open } from "lmdb"; +import { g } from "utils/global"; + +const emptySnapshot = { + type: "" as "" | "comp" | "page", + id: "", + bin: new Uint8Array(), + url: "", + name: "", + ts: Date.now(), +}; +export type DocSnapshot = typeof emptySnapshot; + +export const snapshot = { + _db: null as null | RootDatabase, + init() { + this._db = open({ + name: "user-conf", + path: dir.path(`${g.datadir}/lmdb/doc-snapshot.lmdb`), + }); + return this._db; + }, + get db() { + if (!this._db) { + this._db = this.init(); + } + return this._db; + }, + getOrCreate(data: DocSnapshot) { + const id = `${data.type}-${data.id}`; + let res = this.db.get(id); + if (!res) { + this.db.put(id, structuredClone(emptySnapshot)); + res = this.db.get(id); + } + return res as DocSnapshot; + }, + get(type: string, id: string) { + return this.db.get(`${type}-${id}`); + }, + set(data: DocSnapshot) { + const id = `${data.type}-${data.id}`; + this.db.put(id, data); + }, +}; diff --git a/app/srv/ws/sync/user.ts b/app/srv/ws/sync/entity/user.ts similarity index 88% rename from app/srv/ws/sync/user.ts rename to app/srv/ws/sync/entity/user.ts index c602944b..53fd594f 100644 --- a/app/srv/ws/sync/user.ts +++ b/app/srv/ws/sync/entity/user.ts @@ -16,13 +16,11 @@ export const user = { name: "user-conf", path: dir.path(`${g.datadir}/lmdb/user-conf.lmdb`), }); + return this._db; }, get db() { if (!this._db) { - this._db = open({ - name: "user-conf", - path: dir.path(`${g.datadir}/lmdb/user-conf.lmdb`), - }); + this._db = this.init(); } return this._db; }, diff --git a/app/srv/ws/sync/sync-handler.ts b/app/srv/ws/sync/sync-handler.ts index 18c338ba..fe644a40 100644 --- a/app/srv/ws/sync/sync-handler.ts +++ b/app/srv/ws/sync/sync-handler.ts @@ -7,7 +7,7 @@ import { loadDefaultSite } from "./editor/load"; import { ActionCtx, SyncType } from "./type"; import { SyncActionPaths } from "./actions-def"; import * as actions from "./actions/index"; -import { UserConf, user } from "./user"; +import { UserConf, user } from "./entity/user"; const packr = new Packr({ structuredClone: true }); const conns = new Map< @@ -79,17 +79,23 @@ export const syncHandler: WebSocketHandler = { const code = msg.code as keyof typeof SyncActionPaths; const actionName = SyncActionPaths[code].replace(/\./gi, "_"); if (actionName) { - const action = (actions as any)[actionName].bind({ - user: { id: conn.user_id, conf: conn.conf }, - } as ActionCtx); + const baseAction = (actions as any)[actionName]; + if (!baseAction) { + console.log(`app/ws/edit/sync/${actionName}.ts not found}`); + } + if (baseAction) { + const action = baseAction.bind({ + user: { id: conn.user_id, conf: conn.conf }, + } as ActionCtx); - ws.sendBinary( - packr.pack({ - type: SyncType.ActionResult, - argid: msg.argid, - val: await action(...msg.args), - }) - ); + ws.sendBinary( + packr.pack({ + type: SyncType.ActionResult, + argid: msg.argid, + val: await action(...msg.args), + }) + ); + } } } } diff --git a/app/srv/ws/sync/type.ts b/app/srv/ws/sync/type.ts index 4360da65..ec86e988 100644 --- a/app/srv/ws/sync/type.ts +++ b/app/srv/ws/sync/type.ts @@ -1,4 +1,4 @@ -import { UserConf } from "./user"; +import { UserConf } from "./entity/user"; export enum SyncType { ClientID, diff --git a/app/web/package.json b/app/web/package.json index b8ae8980..40aa1866 100644 --- a/app/web/package.json +++ b/app/web/package.json @@ -22,6 +22,7 @@ "esbuild-wasm": "^0.19.4", "hash-wasm": "^4.10.0", "idb-keyval": "^6.2.1", + "immer": "^10.0.3", "js-base64": "^3.7.5", "lodash.capitalize": "^4.2.1", "lodash.concat": "^4.5.0", diff --git a/app/web/src/base/page/all.tsx b/app/web/src/base/page/all.tsx index f41c0817..824638e1 100644 --- a/app/web/src/base/page/all.tsx +++ b/app/web/src/base/page/all.tsx @@ -1,8 +1,8 @@ import { useEffect } from "react"; import { page, useGlobal } from "web-utils"; -import { Loading } from "../../utils/ui/loading"; -import { bootEd } from "../../render/ed/ed"; import { EDGlobal } from "../../render/ed/logic/ed-global"; +import { edInitSync } from "../../render/ed/logic/ed-sync"; +import { Loading } from "../../utils/ui/loading"; export default page({ url: "**", @@ -14,7 +14,7 @@ export default page({ location.pathname === "/ed" || location.pathname.startsWith("/ed/") ) { - bootEd(p); + edInitSync(p); } else if (location.pathname.startsWith("/editor")) { const arr = location.pathname.split("/"); if (arr.length <= 2) { diff --git a/app/web/src/base/page/ed.tsx b/app/web/src/base/page/ed.tsx index cef1d024..085af4e7 100644 --- a/app/web/src/base/page/ed.tsx +++ b/app/web/src/base/page/ed.tsx @@ -1,17 +1,19 @@ import { page, useGlobal } from "web-utils"; -import { Ed, bootEd } from "../../render/ed/ed"; +import { EdBase } from "../../render/ed/ed-base"; import { EDGlobal } from "../../render/ed/logic/ed-global"; import { Loading } from "../../utils/ui/loading"; +import { initSync } from "wasm-gzip"; +import { edInitSync } from "../../render/ed/logic/ed-sync"; export default page({ url: "/ed/:site_id/:page_id", component: ({}) => { const p = useGlobal(EDGlobal, "EDITOR"); - if (!bootEd(p)) { - return ; + if (!edInitSync(p)) { + return ; } - return ; + return ; }, }); diff --git a/app/web/src/index.tsx b/app/web/src/index.tsx index 4bd82cf1..6bba11b9 100644 --- a/app/web/src/index.tsx +++ b/app/web/src/index.tsx @@ -4,6 +4,9 @@ import { Root } from "./base/root"; import "./index.css"; import { createAPI, createDB, reloadDBAPI } from "./utils/script/init-api"; import { w } from "./utils/types/general"; +import * as Y from "yjs"; + +(window as any).Y = Y; const start = async () => { const base = `${location.protocol}//${location.host}`; diff --git a/app/web/src/render/ed/ed-base.tsx b/app/web/src/render/ed/ed-base.tsx new file mode 100644 index 00000000..fbe8ecb0 --- /dev/null +++ b/app/web/src/render/ed/ed-base.tsx @@ -0,0 +1,26 @@ +import { useGlobal } from "web-utils"; +import { Loading } from "../../utils/ui/loading"; +import { EDGlobal } from "./logic/ed-global"; +import { edRoute } from "./logic/ed-route"; + +export const EdBase = () => { + const p = useGlobal(EDGlobal, "EDITOR"); + + if (p.status === "init") { + p.status = "ready"; + } + + edRoute(p); + + if (p.status === "loading") { + return ; + } + if (p.status === "site-not-found" || p.status === "page-not-found") { + return ( +
+ {p.status === "site-not-found" ? "Site not found" : "Page not found"} +
+ ); + } + return
Editor
; +}; diff --git a/app/web/src/render/ed/logic/ed-global.ts b/app/web/src/render/ed/logic/ed-global.ts index 10d983ff..058af517 100644 --- a/app/web/src/render/ed/logic/ed-global.ts +++ b/app/web/src/render/ed/logic/ed-global.ts @@ -1,4 +1,6 @@ import { clientStartSync } from "../../../utils/sync/client"; +import { IItem } from "../../../utils/types/item"; +import { DPage, IRoot } from "../../../utils/types/root"; const EmptySite = { id: "", @@ -9,8 +11,13 @@ const EmptySite = { config: { api_url: "" }, }; export type ESite = typeof EmptySite; +export type EPage = typeof EmptyPage; + const EmptyPage = { id: "", + name: "", + url: "", + snapshot: null as null | Uint8Array, }; export const EDGlobal = { @@ -22,7 +29,11 @@ export const EDGlobal = { | "ready", sync: null as unknown as Awaited>, site: EmptySite, - page: EmptyPage, + page: { + current: EmptyPage, + doc: null as null | DPage, + root: null as null | IRoot, + }, }; export type PG = typeof EDGlobal & { render: () => void }; diff --git a/app/web/src/render/ed/logic/ed-route.ts b/app/web/src/render/ed/logic/ed-route.ts index cdcf6312..23a585e3 100644 --- a/app/web/src/render/ed/logic/ed-route.ts +++ b/app/web/src/render/ed/logic/ed-route.ts @@ -1,7 +1,8 @@ +import { IRoot } from "../../../utils/types/root"; import { PG } from "./ed-global"; - +import { produce } from "immer"; export const edRoute = async (p: PG) => { - if (p.status === "init") { + if (p.status === "ready") { if (!p.site.domain && !p.site.name) { p.status = "loading"; const site = await p.sync.site.load(p.site.id); @@ -14,8 +15,31 @@ export const edRoute = async (p: PG) => { p.site = site; } - if (p.site) { - console.log(p.site); + if (p.page.current.id !== params.page_id || !p.page.current.snapshot) { + p.status = "loading"; + const page = await p.sync.page.load(params.page_id); + + if (!page) { + p.status = "page-not-found"; + p.render(); + return; + } + + p.page.current = page; + if (page.snapshot) { + const doc = new Y.Doc(); + Y.applyUpdate(doc, page.snapshot); + p.page.doc = doc as any; + + if (p.page.doc) { + const root = p.page.doc.getMap("map").get("root")?.toJSON() as IRoot; + if (root) { + p.page.root = produce(root, () => {}); + } + } + } + p.status = "ready"; + p.render(); } } }; diff --git a/app/web/src/render/ed/ed.tsx b/app/web/src/render/ed/logic/ed-sync.tsx similarity index 58% rename from app/web/src/render/ed/ed.tsx rename to app/web/src/render/ed/logic/ed-sync.tsx index cc47ab4c..51a8e98e 100644 --- a/app/web/src/render/ed/ed.tsx +++ b/app/web/src/render/ed/logic/ed-sync.tsx @@ -1,28 +1,8 @@ -import { useGlobal } from "web-utils"; -import { Loading } from "../../utils/ui/loading"; -import { EDGlobal, PG } from "./logic/ed-global"; -import { edRoute } from "./logic/ed-route"; -import { clientStartSync } from "../../utils/sync/client"; +import { clientStartSync } from "../../../utils/sync/client"; +import { Loading } from "../../../utils/ui/loading"; +import { PG } from "./ed-global"; -export const Ed = () => { - const p = useGlobal(EDGlobal, "EDITOR"); - - edRoute(p); - - if (p.status === "loading") { - return ; - } - if (p.status === "site-not-found" || p.status === "page-not-found") { - return ( -
- {p.status === "site-not-found" ? "Site not found" : "Page not found"} -
- ); - } - return
asfa
; -}; - -export const bootEd = (p: PG) => { +export const edInitSync = (p: PG) => { const session = JSON.parse( localStorage.getItem("prasi-session") || "null" ) as { data: { user: { id: string } } }; diff --git a/app/web/src/render/ed/logic/tree/build.tsx b/app/web/src/render/ed/logic/tree/build.tsx new file mode 100644 index 00000000..754dfd6b --- /dev/null +++ b/app/web/src/render/ed/logic/tree/build.tsx @@ -0,0 +1,5 @@ +import { IItem } from "../../../../utils/types/item"; +import { ISection } from "../../../../utils/types/section"; +import { PG } from "../ed-global"; + +export const treeRebuild = (items: (ISection | IItem)[]) => {}; diff --git a/app/web/src/utils/sync/client.ts b/app/web/src/utils/sync/client.ts index bd701a72..c13d2cbf 100644 --- a/app/web/src/utils/sync/client.ts +++ b/app/web/src/utils/sync/client.ts @@ -4,7 +4,10 @@ import { UseStore, get, set } from "idb-keyval"; import { Packr } from "msgpackr"; import { stringify } from "safe-stable-stringify"; import { SyncActions } from "../../../../srv/ws/sync/actions"; -import { SyncActionDefinition } from "../../../../srv/ws/sync/actions-def"; +import { + SyncActionDefinition, + SyncActionPaths, +} from "../../../../srv/ws/sync/actions-def"; import { initIDB } from "./idb"; import { SyncType } from "../../../../srv/ws/sync/type"; import { w } from "../types/general"; @@ -48,31 +51,23 @@ export const clientStartSync = async (arg: { const { user_id, events } = arg; conf.idb = initIDB(user_id); await connect(user_id, events); - const path: any[] = []; return new DeepProxy( SyncActionDefinition, - ({ trapName, value, key, DEFAULT, PROXY }) => { + ({ target, trapName, value, key, DEFAULT, PROXY }) => { if (trapName === "set") { throw new TypeError("target is immutable"); } - path.push(key); if (typeof value === "string") { - for (let i = 0; i < path.length; i++) { - if (path[i] !== "then") { - path.splice(0, i); - break; - } - } - return (...args: any[]) => - new Promise((resolve) => { + return (...args: any[]) => { + return new Promise((resolve) => { doAction({ - path: path.join("."), code: value, resolve, args, }); }); + }; } if (trapName === "get") { @@ -168,16 +163,16 @@ const loadEventOffline = async (name: ClientEvent) => { }; const doAction = async (arg: { - path: string; code: string; resolve: (value: any) => void; args: any[]; }) => { - const { path, args, code, resolve } = arg; + const { args, code, resolve } = arg; const ws = conf.ws; const idb = conf.idb; if (idb) { const sargs = stringify(args); + const path = (SyncActionPaths as any)[code]; const argid = await xxhash32(`op-${path}-${sargs}`); if (ws && ws.readyState === ws.OPEN) { @@ -186,6 +181,7 @@ const doAction = async (arg: { ts: Date.now(), resolve, }; + ws.send(packr.pack({ type: SyncType.Action, code, args, argid })); } else { // offline diff --git a/app/web/src/utils/types/root.ts b/app/web/src/utils/types/root.ts index bd9b3c96..24ac0886 100644 --- a/app/web/src/utils/types/root.ts +++ b/app/web/src/utils/types/root.ts @@ -1,4 +1,4 @@ -import { TypedArray, TypedMap } from "yjs-types"; +import { TypedArray, TypedDoc, TypedMap } from "yjs-types"; import { ISection } from "./section"; export type IRoot = { @@ -12,3 +12,5 @@ export type MRoot = TypedMap<{ type: "root"; childs: TypedArray; }>; + +export type DPage = TypedDoc<{ map: TypedMap<{ id: string; root: MRoot }> }>; diff --git a/bun.lockb b/bun.lockb index 5c9da669..4f5fd8bc 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/pkgs/core/index.ts b/pkgs/core/index.ts index ab91bcda..519707b8 100644 --- a/pkgs/core/index.ts +++ b/pkgs/core/index.ts @@ -9,7 +9,8 @@ import { g } from "./utils/global"; import { createLogger } from "./utils/logger"; import { preparePrisma } from "./utils/prisma"; import { syncActionDefinition } from "utils/sync-def"; -import { user } from "../../app/srv/ws/sync/user"; +import { user } from "../../app/srv/ws/sync/entity/user"; +import { snapshot } from "../../app/srv/ws/sync/entity/snapshot"; g.status = "init"; @@ -24,7 +25,10 @@ if (g.mode === "dev") { await startDevWatcher(); } +/** init lmdb */ user.conf.init(); +snapshot.init(); + await preparePrisma(); await ensureNotRunning(); @@ -37,8 +41,8 @@ if (g.db) { await syncActionDefinition(); await generateAPIFrm(); await prepareApiRoutes(); -await createServer(); await prepareAPITypes(); await parcelBuild(); +await createServer(); g.status = "ready"; diff --git a/pkgs/web-utils/src/global.ts b/pkgs/web-utils/src/global.ts index 6759005c..ffb46b20 100644 --- a/pkgs/web-utils/src/global.ts +++ b/pkgs/web-utils/src/global.ts @@ -1,5 +1,6 @@ import goober from "goober"; import type { PrismaClient } from "../../../app/db/db"; +import * as Yjs from "yjs"; declare global { const navigate: (path: string) => void; const params: any; @@ -9,5 +10,6 @@ declare global { const db: PrismaClient; const prasiContext: any; const serverurl: string; + const Y: typeof Yjs; } export {};