fixing hex

2024-07-29 17:58:55 +07:00 · 2024-07-29 17:58:55 +07:00 · 1f15dbe2bb
parent fad1e6ffc1
commit 1f15dbe2bb
8 changed files with 330 additions and 30 deletions
--- a/bun.lockb
+++ b/bun.lockb
--- a/pkgs/api/_prasi.ts
+++ b/pkgs/api/_prasi.ts
@ -23,13 +23,13 @@ export const _ = {
      },
      compress: async () => {
        const last = parts.pop();
-        if (last === 'all') {
+        if (last === "all") {
          g.compress.mode = "all";
        }
-        if (last === 'only-gz') {
+        if (last === "only-gz") {
          g.compress.mode = "only-gz";
        }
-        if (last === 'off') {
+        if (last === "off") {
          g.compress.mode = "off";
        }
      },
@ -73,7 +73,7 @@ export const _ = {
      page: async () => {
        const page = g.deploy.pages[parts[1]];
        if (page) {
-          return createResponse(
+          const res = createResponse(
            JSON.stringify({
              id: page.id,
              root: page.content_tree,
@ -84,6 +84,7 @@ export const _ = {
              high_compression: true,
            }
          );
+          return res;
        }
      },
      pages: async () => {
--- a/pkgs/package.json
+++ b/pkgs/package.json
@ -9,6 +9,7 @@
    "execa": "^8.0.1",
    "fs-jetpack": "^5.1.0",
    "mime": "^3.0.0",
+    "msgpackr": "^1.11.0",
    "parse-multipart-data": "^1.5.0",
    "pino": "^8.15.3",
    "pino-pretty": "^10.2.0",
--- a/pkgs/server/api-ctx.ts
+++ b/pkgs/server/api-ctx.ts
@ -1,6 +1,8 @@
 import { simpleHash } from "utils/cache";
 import { g } from "utils/global";
 import { loadCachedBr } from "utils/br-load";
+import { binaryExtensions } from "./binary-ext";
+import mime from "mime";

 const parseQueryParams = (ctx: any) => {
  const pageHref = ctx.req.url;
@ -60,7 +62,13 @@ export const createResponse = (
  const status =
    typeof opt?.res?._status === "number" ? opt?.res?._status : undefined;

-  let content: any = typeof body === "string" ? body : JSON.stringify(body);
+  const content_type = opt?.headers?.["content-type"];
+  const is_binary = binaryExtensions.includes(
+    mime.getExtension(content_type) || ""
+  );
+  let content: any =
+    typeof body === "string" || is_binary ? body : JSON.stringify(body);
+
  const headers = { ...(opt?.headers || {}) } as Record<string, string>;

  if (opt?.cache_accept) {
@ -71,10 +79,6 @@ export const createResponse = (
    ) {
      const content_hash = simpleHash(content);

-      if (!g.cache.br[content_hash]) {
-        loadCachedBr(content_hash, content);
-      }
-
      if (g.cache.br[content_hash]) {
        cached = true;
        content = g.cache.br[content_hash];
@ -85,11 +89,9 @@ export const createResponse = (
    if (opt?.high_compression) {
      if (!cached && opt.cache_accept.toLowerCase().includes("gz")) {
        const content_hash = simpleHash(content);
-
        if (!g.cache.gz[content_hash]) {
          g.cache.gz[content_hash] = Bun.gzipSync(content);
        }
-
        if (g.cache.gz[content_hash]) {
          cached = true;
          content = g.cache.gz[content_hash];
@ -111,6 +113,12 @@ export const createResponse = (
      : undefined
  );

+  if (opt?.headers?.["content-type"]?.includes("woff")) {
+    new Response(body, {
+      headers: { "content-type": headers["content-type"] },
+    });
+  }
+
  for (const [k, v] of Object.entries(headers)) {
    res.headers.append(k, v);
  }
--- a/pkgs/server/binary-ext.ts
+++ b/pkgs/server/binary-ext.ts
@ -0,0 +1,263 @@
+export const binaryExtensions = [
+  "3dm",
+  "3ds",
+  "3g2",
+  "3gp",
+  "7z",
+  "a",
+  "aac",
+  "adp",
+  "afdesign",
+  "afphoto",
+  "afpub",
+  "ai",
+  "aif",
+  "aiff",
+  "alz",
+  "ape",
+  "apk",
+  "appimage",
+  "ar",
+  "arj",
+  "asf",
+  "au",
+  "avi",
+  "bak",
+  "baml",
+  "bh",
+  "bin",
+  "bk",
+  "bmp",
+  "btif",
+  "bz2",
+  "bzip2",
+  "cab",
+  "caf",
+  "cgm",
+  "class",
+  "cmx",
+  "cpio",
+  "cr2",
+  "cur",
+  "dat",
+  "dcm",
+  "deb",
+  "dex",
+  "djvu",
+  "dll",
+  "dmg",
+  "dng",
+  "doc",
+  "docm",
+  "docx",
+  "dot",
+  "dotm",
+  "dra",
+  "DS_Store",
+  "dsk",
+  "dts",
+  "dtshd",
+  "dvb",
+  "dwg",
+  "dxf",
+  "ecelp4800",
+  "ecelp7470",
+  "ecelp9600",
+  "egg",
+  "eol",
+  "eot",
+  "epub",
+  "exe",
+  "f4v",
+  "fbs",
+  "fh",
+  "fla",
+  "flac",
+  "flatpak",
+  "fli",
+  "flv",
+  "fpx",
+  "fst",
+  "fvt",
+  "g3",
+  "gh",
+  "gif",
+  "graffle",
+  "gz",
+  "gzip",
+  "h261",
+  "h263",
+  "h264",
+  "icns",
+  "ico",
+  "ief",
+  "img",
+  "ipa",
+  "iso",
+  "jar",
+  "jpeg",
+  "jpg",
+  "jpgv",
+  "jpm",
+  "jxr",
+  "key",
+  "ktx",
+  "lha",
+  "lib",
+  "lvp",
+  "lz",
+  "lzh",
+  "lzma",
+  "lzo",
+  "m3u",
+  "m4a",
+  "m4v",
+  "mar",
+  "mdi",
+  "mht",
+  "mid",
+  "midi",
+  "mj2",
+  "mka",
+  "mkv",
+  "mmr",
+  "mng",
+  "mobi",
+  "mov",
+  "movie",
+  "mp3",
+  "mp4",
+  "mp4a",
+  "mpeg",
+  "mpg",
+  "mpga",
+  "mxu",
+  "nef",
+  "npx",
+  "numbers",
+  "nupkg",
+  "o",
+  "odp",
+  "ods",
+  "odt",
+  "oga",
+  "ogg",
+  "ogv",
+  "otf",
+  "ott",
+  "pages",
+  "pbm",
+  "pcx",
+  "pdb",
+  "pdf",
+  "pea",
+  "pgm",
+  "pic",
+  "png",
+  "pnm",
+  "pot",
+  "potm",
+  "potx",
+  "ppa",
+  "ppam",
+  "ppm",
+  "pps",
+  "ppsm",
+  "ppsx",
+  "ppt",
+  "pptm",
+  "pptx",
+  "psd",
+  "pya",
+  "pyc",
+  "pyo",
+  "pyv",
+  "qt",
+  "rar",
+  "ras",
+  "raw",
+  "resources",
+  "rgb",
+  "rip",
+  "rlc",
+  "rmf",
+  "rmvb",
+  "rpm",
+  "rtf",
+  "rz",
+  "s3m",
+  "s7z",
+  "scpt",
+  "sgi",
+  "shar",
+  "snap",
+  "sil",
+  "sketch",
+  "slk",
+  "smv",
+  "snk",
+  "so",
+  "stl",
+  "suo",
+  "sub",
+  "swf",
+  "tar",
+  "tbz",
+  "tbz2",
+  "tga",
+  "tgz",
+  "thmx",
+  "tif",
+  "tiff",
+  "tlz",
+  "ttc",
+  "ttf",
+  "txz",
+  "udf",
+  "uvh",
+  "uvi",
+  "uvm",
+  "uvp",
+  "uvs",
+  "uvu",
+  "viv",
+  "vob",
+  "war",
+  "wav",
+  "wax",
+  "wbmp",
+  "wdp",
+  "weba",
+  "webm",
+  "webp",
+  "whl",
+  "wim",
+  "wm",
+  "wma",
+  "wmv",
+  "wmx",
+  "woff",
+  "woff2",
+  "wrm",
+  "wvx",
+  "xbm",
+  "xif",
+  "xla",
+  "xlam",
+  "xls",
+  "xlsb",
+  "xlsm",
+  "xlsx",
+  "xlt",
+  "xltm",
+  "xltx",
+  "xm",
+  "xmind",
+  "xpi",
+  "xpm",
+  "xwd",
+  "xz",
+  "z",
+  "zip",
+  "zipx",
+];
--- a/pkgs/server/serve-web.ts
+++ b/pkgs/server/serve-web.ts
@ -7,9 +7,10 @@ export const serveWeb = async (arg: {
  cache_accept: string;
 }) => {
  const type = mime.getType(arg.pathname);
+
  return createResponse(arg.content, {
    cache_accept: arg.cache_accept,
-    high_compression: true,
+    high_compression: false,
    headers: !type ? undefined : { "content-type": type },
  });
 };
--- a/pkgs/utils/cache.ts
+++ b/pkgs/utils/cache.ts
@ -1,15 +1,28 @@
-export const simpleHash = (str: string, seed = 0) => {
-  let h1 = 0xdeadbeef ^ seed,
-    h2 = 0x41c6ce57 ^ seed;
-  for (let i = 0, ch; i < str.length; i++) {
-    ch = str.charCodeAt(i);
-    h1 = Math.imul(h1 ^ ch, 2654435761);
-    h2 = Math.imul(h2 ^ ch, 1597334677);
-  }
-  h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507);
-  h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909);
-  h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507);
-  h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909);
+import crypto from "crypto";

-  return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString();
+function createHash(data: string, len: any) {
+  return crypto
+    .createHash("shake256", { outputLength: len })
+    .update(data)
+    .digest("hex");
+}
+
+export const simpleHash = (str: string, seed = 0) => {
+  if (typeof str === "string") {
+    let h1 = 0xdeadbeef ^ seed,
+      h2 = 0x41c6ce57 ^ seed;
+    for (let i = 0, ch; i < str.length; i++) {
+      ch = str.charCodeAt(i);
+      h1 = Math.imul(h1 ^ ch, 2654435761);
+      h2 = Math.imul(h2 ^ ch, 1597334677);
+    }
+    h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507);
+    h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909);
+    h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507);
+    h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909);
+
+    return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString();
+  }
+  const hashed = createHash(str, 2);
+  return hashed;
 };
--- a/pkgs/utils/deploy.ts
+++ b/pkgs/utils/deploy.ts
@ -12,6 +12,7 @@ import { gunzipAsync } from "./gzip";
 import { createRouter } from "radix3";
 import { prodIndex } from "./prod-index";
 import { startBrCompress } from "./br-load";
+import { decode } from "msgpackr";

 const decoder = new TextDecoder();
 export const deploy = {
@ -28,15 +29,25 @@ export const deploy = {
    console.log(`Loading site: ${this.config.site_id} ${ts}`);

    try {
-      g.deploy.content = JSON.parse(
-        decoder.decode(
+      if (await Bun.file(`app/web/deploy/${ts}.mpack`).exists()) {
+        g.deploy.content = decode(
          await gunzipAsync(
            new Uint8Array(
              await Bun.file(dir(`app/web/deploy/${ts}.gz`)).arrayBuffer()
            )
          )
-        )
-      );
+        );
+      } else {
+        g.deploy.content = JSON.parse(
+          decoder.decode(
+            await gunzipAsync(
+              new Uint8Array(
+                await Bun.file(dir(`app/web/deploy/${ts}.gz`)).arrayBuffer()
+              )
+            )
+          )
+        );
+      }

      if (g.deploy.content) {
        g.cache = {
@ -116,6 +127,7 @@ export const deploy = {
      }
    } catch (e) {
      console.log("Failed to load site", this.config.site_id);
+      console.error(e);
    }
  },
  async run() {
@ -133,12 +145,13 @@ export const deploy = {
      `Downloading site deploy: ${this.config.site_id} [ts: ${this.config.deploy.ts}]`
    );
    const res = await fetch(
-      `${base_url}/prod-zip/${this.config.site_id}?ts=${Date.now()}`
+      `${base_url}/prod-zip/${this.config.site_id}?ts=${Date.now()}&msgpack=1`
    );
    const ts = Date.now();

    const file = Bun.file(dir(`app/web/deploy/${ts}.gz`));
    await Bun.write(file, await res.arrayBuffer());
+    await Bun.write(dir(`app/web/deploy/${ts}.mpack`), "ok");
    this.config.deploy.ts = ts + "";

    await this.saveConfig();