diff --git a/bun.lockb b/bun.lockb index 6b32bda..38c3991 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/dockerzip b/dockerzip index 547c1fa..752b28f 100644 Binary files a/dockerzip and b/dockerzip differ diff --git a/pkgs/api/_deploy.ts b/pkgs/api/_deploy.ts index 45feef5..1139d3d 100644 --- a/pkgs/api/_deploy.ts +++ b/pkgs/api/_deploy.ts @@ -5,7 +5,7 @@ import { apiContext } from "service-srv"; import { dir } from "utils/dir"; import { g } from "utils/global"; import { restartServer } from "utils/restart"; -import { loadWebCache } from "../server/load-web"; + export const _ = { url: "/_deploy", async api( @@ -35,8 +35,6 @@ export const _ = { case "check": return { now: Date.now(), - current: web.current, - deploys: web.deploys, db: { url: g.dburl || "-", }, @@ -121,7 +119,6 @@ datasource db { await fs.promises.writeFile(`${path}/current`, cur.toString()); web.current = cur; web.deploys.push(cur); - await loadWebCache(web.site_id, web.current); } web.deploying = null; @@ -140,7 +137,6 @@ datasource db { if (web.deploys.find((e) => e === cur)) { web.current = cur; await fs.promises.writeFile(`${path}/current`, cur.toString()); - await loadWebCache(web.site_id, web.current); } } catch (e) { web.current = lastcur; diff --git a/pkgs/api/_file.ts b/pkgs/api/_file.ts index b84a4e8..b2b9dbb 100644 --- a/pkgs/api/_file.ts +++ b/pkgs/api/_file.ts @@ -11,37 +11,6 @@ export const _ = { const rpath = decodeURIComponent(req.params._); let res = new Response("NOT FOUND", { status: 404 }); - try { - if (rpath.startsWith("site")) { - if (rpath === "site-html") { - res = new Response(generateIndexHtml("[[base_url]]", "[[site_id]]")); - } - if (rpath === "site-zip") { - const path = dir(`app/static/site.zip`); - res = new Response(Bun.file(path)); - } - if (rpath === "site-md5") { - const path = dir(`app/static/md5`); - res = new Response(Bun.file(path)); - } - } else if (rpath.startsWith("current-")) { - if (rpath.startsWith("current-md5-")) { - const site_id = rpath.substring("current-md5-".length); - const path = dir(`app/web/${site_id}/current`); - res = new Response(Bun.file(path)); - } else { - const site_id = rpath.substring("current-".length); - const path = dir(`app/web/${site_id}/current`); - const id = await Bun.file(path).text(); - if (id) { - const path = dir(`app/web/${site_id}/deploys/${id}`); - res = new Response(Bun.file(path)); - } - } - } - } catch (e) { - res = new Response("NOT FOUND", { status: 404 }); - } const path = dir(`${g.datadir}/upload/${rpath}`); const file = Bun.file(path); diff --git a/pkgs/api/_web.ts b/pkgs/api/_web.ts deleted file mode 100644 index 54f8f52..0000000 --- a/pkgs/api/_web.ts +++ /dev/null @@ -1,79 +0,0 @@ -import mime from "mime"; -import { apiContext } from "service-srv"; -import { g } from "utils/global"; -import { getApiEntry } from "./_prasi"; - -export const _ = { - url: "/_web/:id/**", - async api(id: string, _: string) { - const { req, res } = apiContext(this); - - const web = g.web[id]; - if (web) { - const cache = web.cache; - if (cache) { - const parts = _.split("/"); - - switch (parts[0]) { - case "site": { - res.setHeader("content-type", "application/json"); - if (req.query_parameters["prod"]) { - return { - site: cache.site, - pages: cache.pages.map((e) => { - return { - id: e.id, - url: e.url, - }; - }), - api: getApiEntry(), - }; - } else { - return cache.site; - } - } - case "pages": { - res.setHeader("content-type", "application/json"); - return cache.pages.map((e) => { - return { - id: e.id, - url: e.url, - }; - }); - } - case "page": { - res.setHeader("content-type", "application/json"); - return cache.pages.find((e) => e.id === parts[1]); - } - case "npm-site": { - let path = parts.slice(1).join("/"); - res.setHeader("content-type", mime.getType(path) || "text/plain"); - - if (path === "site.js") { - path = "index.js"; - } - return cache.npm.site[path]; - } - case "npm-page": { - const page_id = parts[1]; - if (cache.npm.pages[page_id]) { - let path = parts.slice(2).join("/"); - res.setHeader("content-type", mime.getType(path) || "text/plain"); - - if (path === "page.js") { - path = "index.js"; - } - return cache.npm.pages[page_id][path]; - } - res.setHeader("content-type", "text/javascript"); - } - case "comp": { - res.setHeader("content-type", "application/json"); - return cache.comps.find((e) => e.id === parts[1]); - } - } - } - } - return req.params; - }, -}; diff --git a/pkgs/index.ts b/pkgs/index.ts index 3f2a9d9..9e95ac3 100644 --- a/pkgs/index.ts +++ b/pkgs/index.ts @@ -1,16 +1,16 @@ +import { $ } from "execa"; +import { dirAsync, existsAsync } from "fs-jetpack"; +import { deploy } from "utils/deploy"; import { startDevWatcher } from "utils/dev-watcher"; +import { dir } from "utils/dir"; import { ensureNotRunning } from "utils/ensure"; import { preparePrisma } from "utils/prisma"; import { generateAPIFrm } from "./server/api-frm"; import { createServer } from "./server/create"; -import { loadWeb } from "./server/load-web"; import { prepareAPITypes } from "./server/prep-api-ts"; import { config } from "./utils/config"; import { g } from "./utils/global"; import { createLogger } from "./utils/logger"; -import { dirAsync, existsAsync } from "fs-jetpack"; -import { dir } from "utils/dir"; -import { $ } from "execa"; g.mode = process.argv.includes("dev") ? "dev" : "prod"; g.datadir = g.mode === "prod" ? "../data" : ".data"; @@ -44,9 +44,10 @@ if (g.db) { await config.init(); -await loadWeb(); - g.log.info(g.mode === "dev" ? "DEVELOPMENT" : "PRODUCTION"); + + +await deploy.init(); if (g.mode === "dev") { await startDevWatcher(); } diff --git a/pkgs/package.json b/pkgs/package.json index 9e9fa29..f9b42b2 100644 --- a/pkgs/package.json +++ b/pkgs/package.json @@ -6,14 +6,15 @@ "@types/mime": "^3.0.2", "@types/unzipper": "^0.10.7", "execa": "^8.0.1", + "rambda": "^9.1.0", "fs-jetpack": "^5.1.0", "lmdb": "^2.8.5", "mime": "^3.0.0", "pino": "^8.15.3", "pino-pretty": "^10.2.0", - "radash": "^11.0.0", "radix3": "^1.1.0", "typescript": "^5.2.2", - "unzipper": "^0.10.14" + "unzipper": "^0.10.14", + "fast-myers-diff": "^3.2.0" } -} +} \ No newline at end of file diff --git a/pkgs/server/api-ctx.ts b/pkgs/server/api-ctx.ts index ec0e334..c631f37 100644 --- a/pkgs/server/api-ctx.ts +++ b/pkgs/server/api-ctx.ts @@ -109,10 +109,18 @@ export const createResponse = ( return res; }; -function simpleHash(str: string) { - let hash = 0; - for (let i = 0; i < str.length; i++) { - hash = ((hash << 5) - hash + str.charCodeAt(i)) | 0; +export const simpleHash = (str: string, seed = 0) => { + let h1 = 0xdeadbeef ^ seed, + h2 = 0x41c6ce57 ^ seed; + for (let i = 0, ch; i < str.length; i++) { + ch = str.charCodeAt(i); + h1 = Math.imul(h1 ^ ch, 2654435761); + h2 = Math.imul(h2 ^ ch, 1597334677); } - return (hash >>> 0).toString(36); -} + h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507); + h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909); + h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507); + h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909); + + return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString(); +}; diff --git a/pkgs/server/create.ts b/pkgs/server/create.ts index 9149814..040188d 100644 --- a/pkgs/server/create.ts +++ b/pkgs/server/create.ts @@ -1,13 +1,11 @@ +import { file } from "bun"; import { inspectAsync, listAsync } from "fs-jetpack"; import { join } from "path"; import { createRouter } from "radix3"; +import { dir } from "../utils/dir"; import { g } from "../utils/global"; import { parseArgs } from "./parse-args"; import { serveAPI } from "./serve-api"; -import { serveWeb } from "./serve-web"; -import { dir } from "../utils/dir"; -import { file } from "bun"; -import { trim } from "radash"; export const createServer = async () => { g.router = createRouter({ strictTrailingSlash: true }); @@ -72,6 +70,9 @@ export const createServer = async () => { }); }; + if (g.deploy.gz?.code.server) { + } + return handle(req); }, }); diff --git a/pkgs/server/load-web.ts b/pkgs/server/load-web.ts index db7ae3c..86f0b09 100644 --- a/pkgs/server/load-web.ts +++ b/pkgs/server/load-web.ts @@ -1,40 +1,5 @@ -import { file } from "bun"; -import { $ } from "execa"; -import { - dirAsync, - existsAsync, - inspectTreeAsync, - readAsync, - removeAsync, - writeAsync, -} from "fs-jetpack"; -import { gunzipSync } from "zlib"; -import { downloadFile } from "../api/_deploy"; -import { dir } from "../utils/dir"; -import { g } from "../utils/global"; const decoder = new TextDecoder(); -export const loadWeb = async () => { - await dirAsync(dir(`app/static`)); -}; - -export const loadWebCache = async (site_id: string, ts: number | string) => { - const web = g.web; - if (web) { - const path = dir(`app/web/deploys/${ts}`); - if (await existsAsync(path)) { - const fileContent = await readAsync(path, "buffer"); - if (fileContent) { - console.log( - `Loading site ${site_id}: ${humanFileSize(fileContent.byteLength)}` - ); - - const res = gunzipSync(fileContent); - } - } - } -}; - function humanFileSize(bytes: any, si = false, dp = 1) { const thresh = si ? 1000 : 1024; diff --git a/pkgs/server/serve-web.ts b/pkgs/server/serve-web.ts index 4afb3f0..6b0f671 100644 --- a/pkgs/server/serve-web.ts +++ b/pkgs/server/serve-web.ts @@ -2,62 +2,8 @@ import { statSync } from "fs"; import { join } from "path"; import { dir } from "utils/dir"; -const index = { - html: "", - css: { - src: null as any, - encoding: "", - }, - isFile: {} as Record, -}; - export const serveWeb = async (url: URL, req: Request) => { - let site_id = ""; - - if (!site_id) { - return false; - } - - const base = dir(`app/static/site`); - - let path = join(base, url.pathname); - - if (url.pathname === "/site_id") { - return new Response(site_id); - } - - if (url.pathname.startsWith("/index.css")) { - if (!index.css.src) { - const res = await fetch("https://prasi.app/index.css"); - index.css.src = await res.arrayBuffer(); - index.css.encoding = res.headers.get("content-encoding") || ""; - } - - return new Response(index.css.src, { - headers: { - "content-type": "text/css", - "content-encoding": index.css.encoding, - }, - }); - } - - try { - if (typeof index.isFile[path] === "undefined") { - const s = statSync(path); - if (s.isFile()) { - index.isFile[path] = true; - return new Response(Bun.file(path)); - } - } else if (index.isFile[path]) { - return new Response(Bun.file(path)); - } - } catch (e) {} - - if (!index.html) { - index.html = generateIndexHtml("", site_id); - } - - return { site_id, index: index.html }; + return {}; }; export const generateIndexHtml = (base_url: string, site_id: string) => { diff --git a/pkgs/utils/deploy.ts b/pkgs/utils/deploy.ts new file mode 100644 index 0000000..d8973dc --- /dev/null +++ b/pkgs/utils/deploy.ts @@ -0,0 +1,118 @@ +import { dirAsync, read } from "fs-jetpack"; +import { dir } from "./dir"; +import { g } from "./global"; +import { gunzipAsync } from "./gzip"; +import { createRouter } from "radix3"; + +const decoder = new TextDecoder(); +export const deploy = { + async init() { + await dirAsync(dir(`app/web/deploy`)); + + if (!(await this.has_gz())) { + await this.run(); + } + + await this.load(this.config.deploy.ts); + }, + async load(ts: string) { + console.log(`Loading site: ${this.config.site_id} [ts: ${ts}]`); + + try { + g.deploy.gz = JSON.parse( + decoder.decode( + await gunzipAsync( + new Uint8Array( + await Bun.file(dir(`app/web/deploy/${ts}.gz`)).arrayBuffer() + ) + ) + ) + ); + + if (g.deploy.gz) { + for (const page of g.deploy.gz.layouts) { + if (page.is_default_layout) { + g.deploy.layout = page.content_tree; + break; + } + } + if (!g.deploy.layout && g.deploy.gz.layouts.length > 0) { + g.deploy.layout = g.deploy.gz.layouts[0].content_tree; + } + + g.deploy.router = createRouter(); + for (const page of g.deploy.gz.pages) { + g.deploy.router.insert(page.url, page); + } + + g.deploy.comps = {}; + for (const comp of g.deploy.gz.comps) { + g.deploy.comps[comp.id] = comp.content_tree; + } + } + } catch (e) { + console.log("Failed to load site", this.config.site_id); + } + }, + async run() { + if (!this.config.site_id) { + console.log("site_id is not found on app/web/config.json"); + return; + } + + let base_url = "https://prasi.avolut.com"; + if (g.mode === "dev") { + base_url = "http://localhost:4550"; + } + + console.log( + `Downloading site deploy: ${this.config.site_id} [ts: ${this.config.deploy.ts}]` + ); + const res = await fetch(`${base_url}/prod-zip/${this.config.site_id}`); + const ts = Date.now(); + + const file = Bun.file(dir(`app/web/deploy/${ts}.gz`)); + await Bun.write(file, await res.arrayBuffer()); + this.config.deploy.ts = ts + ""; + + await this.saveConfig(); + }, + get config() { + if (!g.deploy) { + g.deploy = { + comps: {}, + layout: null, + router: createRouter(), + config: { deploy: { ts: "" }, site_id: "" }, + init: false, + raw: null, + gz: null as any, + }; + } + + if (!g.deploy.init) { + g.deploy.init = true; + g.deploy.raw = read(dir(`app/web/config.json`), "json"); + for (const [k, v] of Object.entries(g.deploy.raw)) { + (g.deploy.config as any)[k] = v; + } + } + + return g.deploy.config; + }, + saveConfig() { + return Bun.write( + Bun.file(dir(`app/web/config.json`)), + JSON.stringify(this.config, null, 2) + ); + }, + has_gz() { + if (this.config.deploy.ts) { + return Bun.file( + dir(`app/web/deploy/${this.config.deploy.ts}.gz`) + ).exists(); + } + + return false; + }, +}; diff --git a/pkgs/utils/diff.ts b/pkgs/utils/diff.ts new file mode 100644 index 0000000..769e929 --- /dev/null +++ b/pkgs/utils/diff.ts @@ -0,0 +1 @@ +export class Diff {} diff --git a/pkgs/utils/global.ts b/pkgs/utils/global.ts index 785665f..c292d8a 100644 --- a/pkgs/utils/global.ts +++ b/pkgs/utils/global.ts @@ -39,4 +39,32 @@ export const g = global as unknown as { js: string; etag: string; }; + deploy: { + init: boolean; + raw: any; + router: RadixRouter<{ url: string; id: string }>; + layout: null | any; + comps: Record; + gz: null | { + layouts: { + id: string; + url: string; + name: true; + content_tree: any; + is_default_layout: true; + }[]; + pages: { id: string; url: string; name: true; content_tree: any }[]; + site: {}; + comps: { id: string; content_tree: true }[]; + code: { + server: Record; + site: Record; + core: Record; + }; + }; + config: { + site_id: string; + deploy: { ts: string }; + }; + }; }; diff --git a/pkgs/utils/prisma.ts b/pkgs/utils/prisma.ts index 1915c5f..5d9dc8c 100644 --- a/pkgs/utils/prisma.ts +++ b/pkgs/utils/prisma.ts @@ -5,8 +5,10 @@ import { g } from "./global"; export const preparePrisma = async () => { if (await existsAsync(dir("app/db/.env"))) { - await $({ cwd: dir(`app/db`) })`bun prisma db pull`; - await $({ cwd: dir(`app/db`) })`bun prisma generate`; + if (g.mode !== "dev") { + await $({ cwd: dir(`app/db`) })`bun prisma db pull`; + await $({ cwd: dir(`app/db`) })`bun prisma generate`; + } try { const { PrismaClient } = await import("../../app/db/db"); g.db = new PrismaClient();