wip fix

pkgs/api/_deploy.ts

@@ -5,7 +5,7 @@ import { apiContext } from "service-srv";
 import { dir } from "utils/dir";
 import { g } from "utils/global";
 import { restartServer } from "utils/restart";
-import { loadWebCache } from "../server/load-web";
+
 export const _ = {
   url: "/_deploy",
   async api(
@@ -35,8 +35,6 @@ export const _ = {
       case "check":
         return {
           now: Date.now(),
-          current: web.current,
-          deploys: web.deploys,
           db: {
             url: g.dburl || "-",
           },
@@ -121,7 +119,6 @@ datasource db {
             await fs.promises.writeFile(`${path}/current`, cur.toString());
             web.current = cur;
             web.deploys.push(cur);
-            await loadWebCache(web.site_id, web.current);
           }
           web.deploying = null;
 
@@ -140,7 +137,6 @@ datasource db {
             if (web.deploys.find((e) => e === cur)) {
               web.current = cur;
               await fs.promises.writeFile(`${path}/current`, cur.toString());
-              await loadWebCache(web.site_id, web.current);
             }
           } catch (e) {
             web.current = lastcur;

pkgs/api/_file.ts

@@ -11,37 +11,6 @@ export const _ = {
     const rpath = decodeURIComponent(req.params._);
 
     let res = new Response("NOT FOUND", { status: 404 });
-    try {
-      if (rpath.startsWith("site")) {
-        if (rpath === "site-html") {
-          res = new Response(generateIndexHtml("[[base_url]]", "[[site_id]]"));
-        }
-        if (rpath === "site-zip") {
-          const path = dir(`app/static/site.zip`);
-          res = new Response(Bun.file(path));
-        }
-        if (rpath === "site-md5") {
-          const path = dir(`app/static/md5`);
-          res = new Response(Bun.file(path));
-        }
-      } else if (rpath.startsWith("current-")) {
-        if (rpath.startsWith("current-md5-")) {
-          const site_id = rpath.substring("current-md5-".length);
-          const path = dir(`app/web/${site_id}/current`);
-          res = new Response(Bun.file(path));
-        } else {
-          const site_id = rpath.substring("current-".length);
-          const path = dir(`app/web/${site_id}/current`);
-          const id = await Bun.file(path).text();
-          if (id) {
-            const path = dir(`app/web/${site_id}/deploys/${id}`);
-            res = new Response(Bun.file(path));
-          }
-        }
-      }
-    } catch (e) {
-      res = new Response("NOT FOUND", { status: 404 });
-    }
 
     const path = dir(`${g.datadir}/upload/${rpath}`);
     const file = Bun.file(path);

pkgs/api/_web.ts

@@ -1,79 +0,0 @@
-import mime from "mime";
-import { apiContext } from "service-srv";
-import { g } from "utils/global";
-import { getApiEntry } from "./_prasi";
-
-export const _ = {
-  url: "/_web/:id/**",
-  async api(id: string, _: string) {
-    const { req, res } = apiContext(this);
-
-    const web = g.web[id];
-    if (web) {
-      const cache = web.cache;
-      if (cache) {
-        const parts = _.split("/");
-
-        switch (parts[0]) {
-          case "site": {
-            res.setHeader("content-type", "application/json");
-            if (req.query_parameters["prod"]) {
-              return {
-                site: cache.site,
-                pages: cache.pages.map((e) => {
-                  return {
-                    id: e.id,
-                    url: e.url,
-                  };
-                }),
-                api: getApiEntry(),
-              };
-            } else {
-              return cache.site;
-            }
-          }
-          case "pages": {
-            res.setHeader("content-type", "application/json");
-            return cache.pages.map((e) => {
-              return {
-                id: e.id,
-                url: e.url,
-              };
-            });
-          }
-          case "page": {
-            res.setHeader("content-type", "application/json");
-            return cache.pages.find((e) => e.id === parts[1]);
-          }
-          case "npm-site": {
-            let path = parts.slice(1).join("/");
-            res.setHeader("content-type", mime.getType(path) || "text/plain");
-
-            if (path === "site.js") {
-              path = "index.js";
-            }
-            return cache.npm.site[path];
-          }
-          case "npm-page": {
-            const page_id = parts[1];
-            if (cache.npm.pages[page_id]) {
-              let path = parts.slice(2).join("/");
-              res.setHeader("content-type", mime.getType(path) || "text/plain");
-
-              if (path === "page.js") {
-                path = "index.js";
-              }
-              return cache.npm.pages[page_id][path];
-            }
-            res.setHeader("content-type", "text/javascript");
-          }
-          case "comp": {
-            res.setHeader("content-type", "application/json");
-            return cache.comps.find((e) => e.id === parts[1]);
-          }
-        }
-      }
-    }
-    return req.params;
-  },
-};

pkgs/index.ts

@@ -1,16 +1,16 @@
+import { $ } from "execa";
+import { dirAsync, existsAsync } from "fs-jetpack";
+import { deploy } from "utils/deploy";
 import { startDevWatcher } from "utils/dev-watcher";
+import { dir } from "utils/dir";
 import { ensureNotRunning } from "utils/ensure";
 import { preparePrisma } from "utils/prisma";
 import { generateAPIFrm } from "./server/api-frm";
 import { createServer } from "./server/create";
-import { loadWeb } from "./server/load-web";
 import { prepareAPITypes } from "./server/prep-api-ts";
 import { config } from "./utils/config";
 import { g } from "./utils/global";
 import { createLogger } from "./utils/logger";
-import { dirAsync, existsAsync } from "fs-jetpack";
-import { dir } from "utils/dir";
-import { $ } from "execa";
 
 g.mode = process.argv.includes("dev") ? "dev" : "prod";
 g.datadir = g.mode === "prod" ? "../data" : ".data";
@@ -44,9 +44,10 @@ if (g.db) {
 
 await config.init();
 
-await loadWeb();
-
 g.log.info(g.mode === "dev" ? "DEVELOPMENT" : "PRODUCTION");
+
+
+await deploy.init();
 if (g.mode === "dev") {
   await startDevWatcher();
 }

pkgs/package.json

@@ -6,14 +6,15 @@
     "@types/mime": "^3.0.2",
     "@types/unzipper": "^0.10.7",
     "execa": "^8.0.1",
+    "rambda": "^9.1.0",
     "fs-jetpack": "^5.1.0",
     "lmdb": "^2.8.5",
     "mime": "^3.0.0",
     "pino": "^8.15.3",
     "pino-pretty": "^10.2.0",
-    "radash": "^11.0.0",
     "radix3": "^1.1.0",
     "typescript": "^5.2.2",
-    "unzipper": "^0.10.14"
+    "unzipper": "^0.10.14",
+    "fast-myers-diff": "^3.2.0"
   }
-}
+}

pkgs/server/api-ctx.ts

@@ -109,10 +109,18 @@ export const createResponse = (
   return res;
 };
 
-function simpleHash(str: string) {
-  let hash = 0;
-  for (let i = 0; i < str.length; i++) {
-    hash = ((hash << 5) - hash + str.charCodeAt(i)) | 0;
+export const simpleHash = (str: string, seed = 0) => {
+  let h1 = 0xdeadbeef ^ seed,
+    h2 = 0x41c6ce57 ^ seed;
+  for (let i = 0, ch; i < str.length; i++) {
+    ch = str.charCodeAt(i);
+    h1 = Math.imul(h1 ^ ch, 2654435761);
+    h2 = Math.imul(h2 ^ ch, 1597334677);
   }
-  return (hash >>> 0).toString(36);
-}
+  h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507);
+  h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909);
+  h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507);
+  h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909);
+
+  return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString();
+};

pkgs/server/create.ts

@@ -1,13 +1,11 @@
+import { file } from "bun";
 import { inspectAsync, listAsync } from "fs-jetpack";
 import { join } from "path";
 import { createRouter } from "radix3";
+import { dir } from "../utils/dir";
 import { g } from "../utils/global";
 import { parseArgs } from "./parse-args";
 import { serveAPI } from "./serve-api";
-import { serveWeb } from "./serve-web";
-import { dir } from "../utils/dir";
-import { file } from "bun";
-import { trim } from "radash";
 
 export const createServer = async () => {
   g.router = createRouter({ strictTrailingSlash: true });
@@ -72,6 +70,9 @@ export const createServer = async () => {
         });
       };
 
+      if (g.deploy.gz?.code.server) {
+      }
+
       return handle(req);
     },
   });

pkgs/server/load-web.ts

@@ -1,40 +1,5 @@
-import { file } from "bun";
-import { $ } from "execa";
-import {
-  dirAsync,
-  existsAsync,
-  inspectTreeAsync,
-  readAsync,
-  removeAsync,
-  writeAsync,
-} from "fs-jetpack";
-import { gunzipSync } from "zlib";
-import { downloadFile } from "../api/_deploy";
-import { dir } from "../utils/dir";
-import { g } from "../utils/global";
 const decoder = new TextDecoder();
 
-export const loadWeb = async () => {
-  await dirAsync(dir(`app/static`));
-};
-
-export const loadWebCache = async (site_id: string, ts: number | string) => {
-  const web = g.web;
-  if (web) {
-    const path = dir(`app/web/deploys/${ts}`);
-    if (await existsAsync(path)) {
-      const fileContent = await readAsync(path, "buffer");
-      if (fileContent) {
-        console.log(
-          `Loading site ${site_id}: ${humanFileSize(fileContent.byteLength)}`
-        );
-
-        const res = gunzipSync(fileContent);
-      }
-    }
-  }
-};
-
 function humanFileSize(bytes: any, si = false, dp = 1) {
   const thresh = si ? 1000 : 1024;
 

pkgs/server/serve-web.ts

@@ -2,62 +2,8 @@ import { statSync } from "fs";
 import { join } from "path";
 import { dir } from "utils/dir";
 
-const index = {
-  html: "",
-  css: {
-    src: null as any,
-    encoding: "",
-  },
-  isFile: {} as Record<string, boolean>,
-};
-
 export const serveWeb = async (url: URL, req: Request) => {
-  let site_id = "";
-
-  if (!site_id) {
-    return false;
-  }
-
-  const base = dir(`app/static/site`);
-
-  let path = join(base, url.pathname);
-
-  if (url.pathname === "/site_id") {
-    return new Response(site_id);
-  }
-
-  if (url.pathname.startsWith("/index.css")) {
-    if (!index.css.src) {
-      const res = await fetch("https://prasi.app/index.css");
-      index.css.src = await res.arrayBuffer();
-      index.css.encoding = res.headers.get("content-encoding") || "";
-    }
-
-    return new Response(index.css.src, {
-      headers: {
-        "content-type": "text/css",
-        "content-encoding": index.css.encoding,
-      },
-    });
-  }
-
-  try {
-    if (typeof index.isFile[path] === "undefined") {
-      const s = statSync(path);
-      if (s.isFile()) {
-        index.isFile[path] = true;
-        return new Response(Bun.file(path));
-      }
-    } else if (index.isFile[path]) {
-      return new Response(Bun.file(path));
-    }
-  } catch (e) {}
-
-  if (!index.html) {
-    index.html = generateIndexHtml("", site_id);
-  }
-
-  return { site_id, index: index.html };
+  return {};
 };
 
 export const generateIndexHtml = (base_url: string, site_id: string) => {

pkgs/utils/deploy.ts

@@ -0,0 +1,118 @@
+import { dirAsync, read } from "fs-jetpack";
+import { dir } from "./dir";
+import { g } from "./global";
+import { gunzipAsync } from "./gzip";
+import { createRouter } from "radix3";
+
+const decoder = new TextDecoder();
+export const deploy = {
+  async init() {
+    await dirAsync(dir(`app/web/deploy`));
+
+    if (!(await this.has_gz())) {
+      await this.run();
+    }
+
+    await this.load(this.config.deploy.ts);
+  },
+  async load(ts: string) {
+    console.log(`Loading site: ${this.config.site_id} [ts: ${ts}]`);
+
+    try {
+      g.deploy.gz = JSON.parse(
+        decoder.decode(
+          await gunzipAsync(
+            new Uint8Array(
+              await Bun.file(dir(`app/web/deploy/${ts}.gz`)).arrayBuffer()
+            )
+          )
+        )
+      );
+
+      if (g.deploy.gz) {
+        for (const page of g.deploy.gz.layouts) {
+          if (page.is_default_layout) {
+            g.deploy.layout = page.content_tree;
+            break;
+          }
+        }
+        if (!g.deploy.layout && g.deploy.gz.layouts.length > 0) {
+          g.deploy.layout = g.deploy.gz.layouts[0].content_tree;
+        }
+
+        g.deploy.router = createRouter();
+        for (const page of g.deploy.gz.pages) {
+          g.deploy.router.insert(page.url, page);
+        }
+
+        g.deploy.comps = {};
+        for (const comp of g.deploy.gz.comps) {
+          g.deploy.comps[comp.id] = comp.content_tree;
+        }
+      }
+    } catch (e) {
+      console.log("Failed to load site", this.config.site_id);
+    }
+  },
+  async run() {
+    if (!this.config.site_id) {
+      console.log("site_id is not found on app/web/config.json");
+      return;
+    }
+
+    let base_url = "https://prasi.avolut.com";
+    if (g.mode === "dev") {
+      base_url = "http://localhost:4550";
+    }
+
+    console.log(
+      `Downloading site deploy: ${this.config.site_id} [ts: ${this.config.deploy.ts}]`
+    );
+    const res = await fetch(`${base_url}/prod-zip/${this.config.site_id}`);
+    const ts = Date.now();
+
+    const file = Bun.file(dir(`app/web/deploy/${ts}.gz`));
+    await Bun.write(file, await res.arrayBuffer());
+    this.config.deploy.ts = ts + "";
+
+    await this.saveConfig();
+  },
+  get config() {
+    if (!g.deploy) {
+      g.deploy = {
+        comps: {},
+        layout: null,
+        router: createRouter(),
+        config: { deploy: { ts: "" }, site_id: "" },
+        init: false,
+        raw: null,
+        gz: null as any,
+      };
+    }
+
+    if (!g.deploy.init) {
+      g.deploy.init = true;
+      g.deploy.raw = read(dir(`app/web/config.json`), "json");
+      for (const [k, v] of Object.entries(g.deploy.raw)) {
+        (g.deploy.config as any)[k] = v;
+      }
+    }
+
+    return g.deploy.config;
+  },
+  saveConfig() {
+    return Bun.write(
+      Bun.file(dir(`app/web/config.json`)),
+      JSON.stringify(this.config, null, 2)
+    );
+  },
+  has_gz() {
+    if (this.config.deploy.ts) {
+      return Bun.file(
+        dir(`app/web/deploy/${this.config.deploy.ts}.gz`)
+      ).exists();
+    }
+
+    return false;
+  },
+};

pkgs/utils/diff.ts

@@ -0,0 +1 @@
+export class Diff {}

pkgs/utils/global.ts

@@ -39,4 +39,32 @@ export const g = global as unknown as {
     js: string;
     etag: string;
   };
+  deploy: {
+    init: boolean;
+    raw: any;
+    router: RadixRouter<{ url: string; id: string }>;
+    layout: null | any;
+    comps: Record<string, any>;
+    gz: null | {
+      layouts: {
+        id: string;
+        url: string;
+        name: true;
+        content_tree: any;
+        is_default_layout: true;
+      }[];
+      pages: { id: string; url: string; name: true; content_tree: any }[];
+      site: {};
+      comps: { id: string; content_tree: true }[];
+      code: {
+        server: Record<string, string>;
+        site: Record<string, string>;
+        core: Record<string, string>;
+      };
+    };
+    config: {
+      site_id: string;
+      deploy: { ts: string };
+    };
+  };
 };

pkgs/utils/prisma.ts

@@ -5,8 +5,10 @@ import { g } from "./global";
 
 export const preparePrisma = async () => {
   if (await existsAsync(dir("app/db/.env"))) {
-    await $({ cwd: dir(`app/db`) })`bun prisma db pull`;
-    await $({ cwd: dir(`app/db`) })`bun prisma generate`;
+    if (g.mode !== "dev") {
+      await $({ cwd: dir(`app/db`) })`bun prisma db pull`;
+      await $({ cwd: dir(`app/db`) })`bun prisma generate`;
+    }
     try {
       const { PrismaClient } = await import("../../app/db/db");
       g.db = new PrismaClient();