diff --git a/bun.lockb b/bun.lockb index 4462814..c394241 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/pkgs/api/_prasi.ts b/pkgs/api/_prasi.ts index 9ecff08..fe1e111 100644 --- a/pkgs/api/_prasi.ts +++ b/pkgs/api/_prasi.ts @@ -23,13 +23,13 @@ export const _ = { }, compress: async () => { const last = parts.pop(); - if (last === 'all') { + if (last === "all") { g.compress.mode = "all"; } - if (last === 'only-gz') { + if (last === "only-gz") { g.compress.mode = "only-gz"; } - if (last === 'off') { + if (last === "off") { g.compress.mode = "off"; } }, @@ -73,7 +73,7 @@ export const _ = { page: async () => { const page = g.deploy.pages[parts[1]]; if (page) { - return createResponse( + const res = createResponse( JSON.stringify({ id: page.id, root: page.content_tree, @@ -84,6 +84,7 @@ export const _ = { high_compression: true, } ); + return res; } }, pages: async () => { diff --git a/pkgs/package.json b/pkgs/package.json index fd748d1..51d9d30 100644 --- a/pkgs/package.json +++ b/pkgs/package.json @@ -9,6 +9,7 @@ "execa": "^8.0.1", "fs-jetpack": "^5.1.0", "mime": "^3.0.0", + "msgpackr": "^1.11.0", "parse-multipart-data": "^1.5.0", "pino": "^8.15.3", "pino-pretty": "^10.2.0", diff --git a/pkgs/server/api-ctx.ts b/pkgs/server/api-ctx.ts index 9f1cfa5..9da260c 100644 --- a/pkgs/server/api-ctx.ts +++ b/pkgs/server/api-ctx.ts @@ -1,6 +1,8 @@ import { simpleHash } from "utils/cache"; import { g } from "utils/global"; import { loadCachedBr } from "utils/br-load"; +import { binaryExtensions } from "./binary-ext"; +import mime from "mime"; const parseQueryParams = (ctx: any) => { const pageHref = ctx.req.url; @@ -60,7 +62,13 @@ export const createResponse = ( const status = typeof opt?.res?._status === "number" ? opt?.res?._status : undefined; - let content: any = typeof body === "string" ? body : JSON.stringify(body); + const content_type = opt?.headers?.["content-type"]; + const is_binary = binaryExtensions.includes( + mime.getExtension(content_type) || "" + ); + let content: any = + typeof body === "string" || is_binary ? body : JSON.stringify(body); + const headers = { ...(opt?.headers || {}) } as Record; if (opt?.cache_accept) { @@ -71,10 +79,6 @@ export const createResponse = ( ) { const content_hash = simpleHash(content); - if (!g.cache.br[content_hash]) { - loadCachedBr(content_hash, content); - } - if (g.cache.br[content_hash]) { cached = true; content = g.cache.br[content_hash]; @@ -85,11 +89,9 @@ export const createResponse = ( if (opt?.high_compression) { if (!cached && opt.cache_accept.toLowerCase().includes("gz")) { const content_hash = simpleHash(content); - if (!g.cache.gz[content_hash]) { g.cache.gz[content_hash] = Bun.gzipSync(content); } - if (g.cache.gz[content_hash]) { cached = true; content = g.cache.gz[content_hash]; @@ -111,6 +113,12 @@ export const createResponse = ( : undefined ); + if (opt?.headers?.["content-type"]?.includes("woff")) { + new Response(body, { + headers: { "content-type": headers["content-type"] }, + }); + } + for (const [k, v] of Object.entries(headers)) { res.headers.append(k, v); } diff --git a/pkgs/server/binary-ext.ts b/pkgs/server/binary-ext.ts new file mode 100644 index 0000000..377735e --- /dev/null +++ b/pkgs/server/binary-ext.ts @@ -0,0 +1,263 @@ +export const binaryExtensions = [ + "3dm", + "3ds", + "3g2", + "3gp", + "7z", + "a", + "aac", + "adp", + "afdesign", + "afphoto", + "afpub", + "ai", + "aif", + "aiff", + "alz", + "ape", + "apk", + "appimage", + "ar", + "arj", + "asf", + "au", + "avi", + "bak", + "baml", + "bh", + "bin", + "bk", + "bmp", + "btif", + "bz2", + "bzip2", + "cab", + "caf", + "cgm", + "class", + "cmx", + "cpio", + "cr2", + "cur", + "dat", + "dcm", + "deb", + "dex", + "djvu", + "dll", + "dmg", + "dng", + "doc", + "docm", + "docx", + "dot", + "dotm", + "dra", + "DS_Store", + "dsk", + "dts", + "dtshd", + "dvb", + "dwg", + "dxf", + "ecelp4800", + "ecelp7470", + "ecelp9600", + "egg", + "eol", + "eot", + "epub", + "exe", + "f4v", + "fbs", + "fh", + "fla", + "flac", + "flatpak", + "fli", + "flv", + "fpx", + "fst", + "fvt", + "g3", + "gh", + "gif", + "graffle", + "gz", + "gzip", + "h261", + "h263", + "h264", + "icns", + "ico", + "ief", + "img", + "ipa", + "iso", + "jar", + "jpeg", + "jpg", + "jpgv", + "jpm", + "jxr", + "key", + "ktx", + "lha", + "lib", + "lvp", + "lz", + "lzh", + "lzma", + "lzo", + "m3u", + "m4a", + "m4v", + "mar", + "mdi", + "mht", + "mid", + "midi", + "mj2", + "mka", + "mkv", + "mmr", + "mng", + "mobi", + "mov", + "movie", + "mp3", + "mp4", + "mp4a", + "mpeg", + "mpg", + "mpga", + "mxu", + "nef", + "npx", + "numbers", + "nupkg", + "o", + "odp", + "ods", + "odt", + "oga", + "ogg", + "ogv", + "otf", + "ott", + "pages", + "pbm", + "pcx", + "pdb", + "pdf", + "pea", + "pgm", + "pic", + "png", + "pnm", + "pot", + "potm", + "potx", + "ppa", + "ppam", + "ppm", + "pps", + "ppsm", + "ppsx", + "ppt", + "pptm", + "pptx", + "psd", + "pya", + "pyc", + "pyo", + "pyv", + "qt", + "rar", + "ras", + "raw", + "resources", + "rgb", + "rip", + "rlc", + "rmf", + "rmvb", + "rpm", + "rtf", + "rz", + "s3m", + "s7z", + "scpt", + "sgi", + "shar", + "snap", + "sil", + "sketch", + "slk", + "smv", + "snk", + "so", + "stl", + "suo", + "sub", + "swf", + "tar", + "tbz", + "tbz2", + "tga", + "tgz", + "thmx", + "tif", + "tiff", + "tlz", + "ttc", + "ttf", + "txz", + "udf", + "uvh", + "uvi", + "uvm", + "uvp", + "uvs", + "uvu", + "viv", + "vob", + "war", + "wav", + "wax", + "wbmp", + "wdp", + "weba", + "webm", + "webp", + "whl", + "wim", + "wm", + "wma", + "wmv", + "wmx", + "woff", + "woff2", + "wrm", + "wvx", + "xbm", + "xif", + "xla", + "xlam", + "xls", + "xlsb", + "xlsm", + "xlsx", + "xlt", + "xltm", + "xltx", + "xm", + "xmind", + "xpi", + "xpm", + "xwd", + "xz", + "z", + "zip", + "zipx", +]; diff --git a/pkgs/server/serve-web.ts b/pkgs/server/serve-web.ts index c72c636..9775808 100644 --- a/pkgs/server/serve-web.ts +++ b/pkgs/server/serve-web.ts @@ -7,9 +7,10 @@ export const serveWeb = async (arg: { cache_accept: string; }) => { const type = mime.getType(arg.pathname); + return createResponse(arg.content, { cache_accept: arg.cache_accept, - high_compression: true, + high_compression: false, headers: !type ? undefined : { "content-type": type }, }); }; diff --git a/pkgs/utils/cache.ts b/pkgs/utils/cache.ts index f964186..78d6a3d 100644 --- a/pkgs/utils/cache.ts +++ b/pkgs/utils/cache.ts @@ -1,15 +1,28 @@ -export const simpleHash = (str: string, seed = 0) => { - let h1 = 0xdeadbeef ^ seed, - h2 = 0x41c6ce57 ^ seed; - for (let i = 0, ch; i < str.length; i++) { - ch = str.charCodeAt(i); - h1 = Math.imul(h1 ^ ch, 2654435761); - h2 = Math.imul(h2 ^ ch, 1597334677); - } - h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507); - h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909); - h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507); - h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909); +import crypto from "crypto"; - return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString(); +function createHash(data: string, len: any) { + return crypto + .createHash("shake256", { outputLength: len }) + .update(data) + .digest("hex"); +} + +export const simpleHash = (str: string, seed = 0) => { + if (typeof str === "string") { + let h1 = 0xdeadbeef ^ seed, + h2 = 0x41c6ce57 ^ seed; + for (let i = 0, ch; i < str.length; i++) { + ch = str.charCodeAt(i); + h1 = Math.imul(h1 ^ ch, 2654435761); + h2 = Math.imul(h2 ^ ch, 1597334677); + } + h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507); + h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909); + h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507); + h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909); + + return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString(); + } + const hashed = createHash(str, 2); + return hashed; }; diff --git a/pkgs/utils/deploy.ts b/pkgs/utils/deploy.ts index ef666c4..339e0ed 100644 --- a/pkgs/utils/deploy.ts +++ b/pkgs/utils/deploy.ts @@ -12,6 +12,7 @@ import { gunzipAsync } from "./gzip"; import { createRouter } from "radix3"; import { prodIndex } from "./prod-index"; import { startBrCompress } from "./br-load"; +import { decode } from "msgpackr"; const decoder = new TextDecoder(); export const deploy = { @@ -28,15 +29,25 @@ export const deploy = { console.log(`Loading site: ${this.config.site_id} ${ts}`); try { - g.deploy.content = JSON.parse( - decoder.decode( + if (await Bun.file(`app/web/deploy/${ts}.mpack`).exists()) { + g.deploy.content = decode( await gunzipAsync( new Uint8Array( await Bun.file(dir(`app/web/deploy/${ts}.gz`)).arrayBuffer() ) ) - ) - ); + ); + } else { + g.deploy.content = JSON.parse( + decoder.decode( + await gunzipAsync( + new Uint8Array( + await Bun.file(dir(`app/web/deploy/${ts}.gz`)).arrayBuffer() + ) + ) + ) + ); + } if (g.deploy.content) { g.cache = { @@ -116,6 +127,7 @@ export const deploy = { } } catch (e) { console.log("Failed to load site", this.config.site_id); + console.error(e); } }, async run() { @@ -133,12 +145,13 @@ export const deploy = { `Downloading site deploy: ${this.config.site_id} [ts: ${this.config.deploy.ts}]` ); const res = await fetch( - `${base_url}/prod-zip/${this.config.site_id}?ts=${Date.now()}` + `${base_url}/prod-zip/${this.config.site_id}?ts=${Date.now()}&msgpack=1` ); const ts = Date.now(); const file = Bun.file(dir(`app/web/deploy/${ts}.gz`)); await Bun.write(file, await res.arrayBuffer()); + await Bun.write(dir(`app/web/deploy/${ts}.mpack`), "ok"); this.config.deploy.ts = ts + ""; await this.saveConfig();