From 9a0523876c67e9f74bf76403bd5932522ce4fcae Mon Sep 17 00:00:00 2001 From: Rizky Date: Mon, 23 Dec 2024 09:45:19 +0700 Subject: [PATCH] checkpoint --- internal/api/_dbs.ts | 35 +++++ internal/api/_deploy.ts | 241 +++++++++++++++++++++++++++++++ internal/api/_file.ts | 148 +++++++++++++++++++ internal/api/_finfo.ts | 60 ++++++++ internal/api/_font.ts | 62 ++++++++ internal/api/_img.ts | 89 ++++++++++++ internal/api/_kv.ts | 60 ++++++++ internal/api/_notif.ts | 101 +++++++++++++ internal/api/_prasi.ts | 187 ++++++++++++++++++++++++ internal/api/_proxy.ts | 47 ++++++ internal/api/_upload.ts | 79 ++++++++++ internal/api/_zip.ts | 144 +++++++++++++++++++ internal/server/server.ts | 23 ++- internal/supervisor.ts | 3 + internal/utils/api-context.ts | 111 ++++++++++++++ internal/utils/binary-ext.ts | 263 ++++++++++++++++++++++++++++++++++ internal/utils/config.ts | 13 +- internal/utils/fs.ts | 5 +- internal/utils/global.ts | 3 +- internal/utils/parse-env.ts | 66 +++++++++ 20 files changed, 1734 insertions(+), 6 deletions(-) create mode 100644 internal/api/_dbs.ts create mode 100644 internal/api/_deploy.ts create mode 100644 internal/api/_file.ts create mode 100644 internal/api/_finfo.ts create mode 100644 internal/api/_font.ts create mode 100644 internal/api/_img.ts create mode 100644 internal/api/_kv.ts create mode 100644 internal/api/_notif.ts create mode 100644 internal/api/_prasi.ts create mode 100644 internal/api/_proxy.ts create mode 100644 internal/api/_upload.ts create mode 100644 internal/api/_zip.ts create mode 100644 internal/utils/api-context.ts create mode 100644 internal/utils/binary-ext.ts create mode 100644 internal/utils/parse-env.ts diff --git a/internal/api/_dbs.ts b/internal/api/_dbs.ts new file mode 100644 index 0000000..bed413c --- /dev/null +++ b/internal/api/_dbs.ts @@ -0,0 +1,35 @@ +import { gunzipSync } from "bun"; +import { unpack } from "msgpackr"; +import { apiContext } from "utils/api-context"; +import { execQuery } from "utils/query"; + +const g = global as any; +export const _ = { + url: "/_dbs/*", + raw: true, + async api() { + const ctx = apiContext(this); + const { req, res } = ctx; + if (typeof g.db !== "undefined") { + if (req.params._ === "check") { + return { mode: "encrypted" }; + } + + try { + const body = unpack(gunzipSync(await req.arrayBuffer())); + + try { + const result = await execQuery(body, g.db); + return result; + } catch (e: any) { + console.log("_dbs error", body, e.message); + res.sendStatus(500); + res.send(e.message); + } + } catch (e) { + res.sendStatus(500); + res.send('{status: "unauthorized"}'); + } + } + }, +}; diff --git a/internal/api/_deploy.ts b/internal/api/_deploy.ts new file mode 100644 index 0000000..2bfa7ed --- /dev/null +++ b/internal/api/_deploy.ts @@ -0,0 +1,241 @@ +import { $ } from "bun"; +import { readdirSync } from "fs"; +import { readAsync, removeAsync, writeAsync } from "fs-jetpack"; +import { apiContext } from "utils/api-context"; +import { config as _config, type SiteConfig } from "utils/config"; +import { fs } from "utils/fs"; +import { genEnv, parseEnv } from "utils/parse-env"; + +export const _ = { + url: "/_deploy", + async api( + action: ( + | { type: "check" } + | { type: "db-update"; url: string; orm: SiteConfig["db"]["orm"] } + | { type: "db-pull" } + | { type: "db-gen" } + | { type: "db-ver" } + | { type: "db-sync"; url: string } + | { type: "restart" } + | { type: "domain-add"; domain: string } + | { type: "domain-del"; domain: string } + | { type: "deploy-del"; ts: string } + | { type: "deploy"; load_from?: string } + | { type: "deploy-status" } + | { type: "redeploy"; ts: string } + ) & { + id_site: string; + } + ) { + const { res, req } = apiContext(this); + const deploy = _config.current?.deploy!; + const config = _config.current!; + + if (typeof req.query_parameters["export"] !== "undefined") { + return new Response( + Bun.file(fs.path(`site:deploy/current/${deploy.current}.gz`)) + ); + } + + switch (action.type) { + case "check": + const deploys = readdirSync(fs.path("site:deploy/history")); + + return { + now: Date.now(), + current: deploy.current, + deploys: deploys + .filter((e) => e.endsWith(".gz")) + .map((e) => parseInt(e.replace(".gz", ""))), + db: { + url: config.db.url, + orm: config.db.orm, + }, + }; + case "db-ver": { + return (await fs.read(`site:app/db/version`, "string")) || ""; + } + case "db-sync": { + const res = await fetch(action.url); + const text = await res.text(); + if (text) { + await Bun.write(fs.path("site:app/db/prisma/schema.prisma"), text); + await Bun.write( + fs.path(`site:app/db/version`), + Date.now().toString() + ); + } + return "ok"; + } + case "db-update": + if (action.url) { + config.db.url = action.url; + config.db.orm = action.orm; + const env = genEnv({ + ...parseEnv(await Bun.file(fs.path("site:app/db/.env")).text()), + DATABASE_URL: action.url, + }); + await Bun.write(fs.path("site:app/db/.env"), env); + } + return "ok"; + case "db-gen": + { + await $`bun prisma generate`.cwd(fs.path("site:app/db")); + + res.send("ok"); + setTimeout(() => { + // restartServer(); + }, 300); + } + break; + case "db-pull": + { + let env = await readAsync(fs.path("site:app/db/.env")); + if (env) { + const ENV = parseEnv(env); + if (typeof ENV.DATABASE_URL === "string") { + const type = ENV.DATABASE_URL.split("://").shift(); + if (type) { + await writeAsync( + fs.path("site:app/db/prisma/schema.prisma"), + `\ + generator client { + provider = "prisma-client-js" + } + + datasource db { + provider = "${type}" + url = env("DATABASE_URL") + }` + ); + + try { + await Bun.write( + fs.path("site:app/db/.env"), + `DATABASE_URL=${ENV.DATABASE_URL}` + ); + await $`bun install`.cwd(fs.path("site:app/db")); + await $`bun prisma db pull --force`.cwd( + fs.path("site:app/db") + ); + await $`bun prisma generate`.cwd(fs.path("site:app/db")); + await Bun.write( + fs.path(`site:app/db/version`), + Date.now().toString() + ); + } catch (e) { + console.error(e); + } + res.send("ok"); + setTimeout(() => { + // restartServer(); + }, 300); + } + } + } + } + break; + case "restart": + { + res.send("ok"); + setTimeout(() => { + // restartServer(); + }, 300); + } + break; + case "deploy-del": + { + await removeAsync(fs.path(`site:deploy/history/${action.ts}.gz`)); + const deploys = readdirSync(fs.path(`site:deploy/history`)); + + return { + now: Date.now(), + current: deploy.current, + deploys: deploys + .filter((e) => e.endsWith(".gz")) + .map((e) => parseInt(e.replace(".gz", ""))), + }; + } + break; + case "deploy-status": + break; + case "deploy": + { + await _config.init("site:site.json"); + await _config.set("site_id", action.id_site); + fs.init(config); + + return { + now: Date.now(), + current: deploy.current, + deploys: config.deploy.history, + }; + } + break; + case "redeploy": + { + // deploy.config.deploy.ts = action.ts; + // await deploy.saveConfig(); + // await deploy.load(action.ts); + // const deploys = fs.readdirSync(dir(`/app/web/deploy`)); + + // return { + // now: Date.now(), + // current: parseInt(deploy.config.deploy.ts), + // deploys: deploys + // .filter((e) => e.endsWith(".gz")) + // .map((e) => parseInt(e.replace(".gz", ""))), + // }; + } + break; + } + }, +}; + +export const downloadFile = async ( + url: string, + filePath: string, + progress?: (rec: number, total: number) => void +) => { + try { + const _url = new URL(url); + if (_url.hostname === "localhost") { + _url.hostname = "127.0.0.1"; + } + // g.log.info(`Downloading ${url} to ${filePath}`); + const res = await fetch(_url as any); + if (res.body) { + const file = Bun.file(filePath); + + const writer = file.writer(); + const reader = res.body.getReader(); + + // Step 3: read the data + let receivedLength = 0; // received that many bytes at the moment + let chunks = []; // array of received binary chunks (comprises the body) + while (true) { + const { done, value } = await reader.read(); + + if (done) { + writer.end(); + break; + } + + chunks.push(value); + writer.write(value); + receivedLength += value.length; + + if (progress) { + progress( + receivedLength, + parseInt(res.headers.get("content-length") || "0") + ); + } + } + } + return true; + } catch (e) { + console.log(e); + return false; + } +}; diff --git a/internal/api/_file.ts b/internal/api/_file.ts new file mode 100644 index 0000000..88e8bcc --- /dev/null +++ b/internal/api/_file.ts @@ -0,0 +1,148 @@ +import mime from "mime"; +import { apiContext } from "utils/api-context"; +import { dir } from "utils/dir"; +import { g } from "utils/global"; +import { readdir, stat } from "fs/promises"; +import { basename, dirname } from "path"; +import { + dirAsync, + existsAsync, + moveAsync, + removeAsync, + renameAsync, +} from "fs-jetpack"; + +export const _ = { + url: "/_file/**", + async api() { + const { req } = apiContext(this); + let rpath = decodeURIComponent(req.params._); + rpath = rpath + .split("/") + .map((e) => e.replace(/\.\./gi, "")) + .filter((e) => !!e) + .join("/"); + + let res = new Response("NOT FOUND", { status: 404 }); + + if (Object.keys(req.query_parameters).length > 0) { + await dirAsync(dir(`${g.datadir}/files`)); + const base_dir = dir(`${g.datadir}/files/${rpath}`); + if (typeof req.query_parameters["move"] === "string") { + if (rpath) { + let moveto = req.query_parameters["move"]; + + moveto = moveto + .split("/") + .map((e) => e.replace(/\.\./gi, "")) + .filter((e) => !!e) + .join("/"); + + await moveAsync( + dir(`${g.datadir}/files/${rpath}`), + dir(`${g.datadir}/files/${moveto}/${basename(rpath)}`) + ); + } + + return new Response(JSON.stringify({ status: "ok" }), { + headers: { "content-type": "application/json" }, + }); + } else if (typeof req.query_parameters["del"] === "string") { + if (rpath) { + const base_dir = dir(`${g.datadir}/files/${rpath}`); + if (await existsAsync(base_dir)) { + const s = await stat(base_dir); + if (s.isDirectory()) { + if ((await readdir(base_dir)).length === 0) { + await removeAsync(base_dir); + } + } else { + await removeAsync(base_dir); + } + } + } + + return new Response(JSON.stringify({ status: "ok" }), { + headers: { "content-type": "application/json" }, + }); + } else if (typeof req.query_parameters["rename"] === "string") { + let rename = req.query_parameters["rename"]; + + rename = rename + .split("/") + .map((e) => e.replace(/\.\./gi, "")) + .filter((e) => !!e) + .join("/"); + + let newname = ""; + if (rpath) { + if (await existsAsync(dir(`${g.datadir}/files/${rpath}`))) { + await renameAsync(dir(`${g.datadir}/files/${rpath}`), rename); + } else { + const target = dir( + `${g.datadir}/files/${dirname(rpath)}/${rename}` + ); + await dirAsync(target); + } + newname = `/${dirname(rpath)}/${rename}`; + } + + return new Response(JSON.stringify({ newname }), { + headers: { "content-type": "application/json" }, + }); + } else if (typeof req.query_parameters["dir"] === "string") { + try { + const files = [] as { + name: string; + type: "dir" | "file"; + size: number; + }[]; + await Promise.all( + ( + await readdir(base_dir) + ).map(async (e) => { + const s = await stat(dir(`${g.datadir}/files/${rpath}/${e}`)); + files.push({ + name: e, + type: s.isDirectory() ? "dir" : "file", + size: s.size, + }); + }) + ); + return new Response(JSON.stringify(files), { + headers: { "content-type": "application/json" }, + }); + } catch (e) { + return new Response(JSON.stringify(null), { + headers: { "content-type": "application/json" }, + }); + } + } + } + + const path = dir(`${g.datadir}/files/${rpath}`); + const file = Bun.file(path); + + if (await file.exists()) { + res = new Response(file); + } else { + res = new Response("NOT FOUND", { status: 404 }); + } + + const arr = path.split("-"); + const ext = arr.pop(); + const fname = arr.join("-") + "." + ext; + const ctype = mime.getType(fname); + if (ctype) { + res.headers.set("content-disposition", `inline; filename="${fname}"`); + res.headers.set("content-type", ctype); + } + + res.headers.set("Access-Control-Allow-Origin", "*"); + res.headers.set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT"); + res.headers.set("Access-Control-Allow-Headers", "content-type"); + res.headers.set("Access-Control-Allow-Credentials", "true"); + + return res; + }, +}; diff --git a/internal/api/_finfo.ts b/internal/api/_finfo.ts new file mode 100644 index 0000000..62c6043 --- /dev/null +++ b/internal/api/_finfo.ts @@ -0,0 +1,60 @@ +import mime from "mime"; +import { apiContext } from "utils/api-context"; +import { fs } from "utils/fs"; + +export const _ = { + url: "/_finfo/**", + async api() { + const { req } = apiContext(this); + let rpath = decodeURIComponent(req.params._); + rpath = rpath + .split("/") + .map((e) => e.replace(/\.\./gi, "")) + .filter((e) => !!e) + .join("/"); + + let res = new Response("NOT FOUND", { status: 404 }); + + const path = fs.path(`upload:${rpath}`); + const file = Bun.file(path); + + if (await file.exists()) { + const arr = (path.split("/").pop() || "").split("-"); + const ctype = mime.getType(path); + const ext = mime.getExtension(ctype || ""); + const fname = arr.join("-") + "." + ext; + + res = new Response( + JSON.stringify({ + filename: fname, + fullpath: path, + size: formatFileSize(file.size), + mime: ctype, + ext, + }), + { + headers: { "content-type": "application/json" }, + } + ); + } else { + res = new Response("null", { + headers: { "content-type": "application/json" }, + }); + } + + return res; + }, +}; + +function formatFileSize(bytes: number) { + const units = ["B", "KB", "MB", "GB", "TB"]; + let size = bytes; + let unitIndex = 0; + + while (size >= 1024 && unitIndex < units.length - 1) { + size /= 1024; + unitIndex++; + } + + return `${size.toFixed(2)} ${units[unitIndex]}`; +} diff --git a/internal/api/_font.ts b/internal/api/_font.ts new file mode 100644 index 0000000..629ce78 --- /dev/null +++ b/internal/api/_font.ts @@ -0,0 +1,62 @@ +import { apiContext } from "utils/api-context"; + +const g = global as unknown as { + _font_cache: Record; +}; + +if (!g._font_cache) { + g._font_cache = {}; +} + +export const _ = { + url: "/_font/**", + async api() { + const { req } = apiContext(this); + const pathname = req.url.split("/_font").pop() || ""; + const cache = g._font_cache[pathname]; + if (cache) { + if (req.headers.get("accept-encoding")?.includes("gzip")) { + return new Response(Bun.gzipSync(cache.body), { + headers: { + "content-type": cache.headers["content-type"], + "content-encoding": "gzip", + }, + }); + } else { + return new Response(cache.body, { + headers: { + "content-type": cache.headers["content-type"], + }, + }); + } + } + let f: Response = null as any; + let raw = false; + if (pathname?.startsWith("/s/")) { + f = await fetch(`https://fonts.gstatic.com${pathname}`); + raw = true; + } else { + f = await fetch(`https://fonts.googleapis.com${pathname}`); + } + if (f) { + let body = null as any; + + if (!raw) { + body = await f.text(); + body = body.replaceAll("https://fonts.gstatic.com", "/_font"); + } else { + body = await f.arrayBuffer(); + } + + g._font_cache[pathname] = { body, headers: {} }; + f.headers.forEach((v, k) => { + g._font_cache[pathname].headers[k] = v; + }); + + const res = new Response(body); + + res.headers.set("content-type", f.headers.get("content-type") || ""); + return res; + } + }, +}; diff --git a/internal/api/_img.ts b/internal/api/_img.ts new file mode 100644 index 0000000..79a7d30 --- /dev/null +++ b/internal/api/_img.ts @@ -0,0 +1,89 @@ +import { dirAsync } from "fs-jetpack"; +import { apiContext } from "utils/api-context"; +import { stat } from "fs/promises"; +import { dir } from "utils/dir"; +import { g } from "utils/global"; +import { dirname, parse } from "path"; +import sharp from "sharp"; + +const modified = {} as Record; + +export const _ = { + url: "/_img/**", + async api() { + const { req } = apiContext(this); + let res = new Response("NOT FOUND", { status: 404 }); + + const w = parseInt(req.query_parameters.w); + const h = parseInt(req.query_parameters.h); + const fit = req.query_parameters.fit; + let force = typeof req.query_parameters.force === "string"; + + let rpath = decodeURIComponent(req.params._); + rpath = rpath + .split("/") + .map((e) => e.replace(/\.\./gi, "")) + .filter((e) => !!e) + .join("/"); + + try { + const filepath = dir(`${g.datadir}/files/${rpath}`); + const st = await stat(filepath); + if (st.isFile()) { + if ( + !modified[filepath] || + (modified[filepath] && modified[filepath] !== st.mtimeMs) + ) { + modified[filepath] = st.mtimeMs; + force = true; + } + + if (!w && !h) { + const file = Bun.file(filepath); + return new Response(file); + } else { + const original = Bun.file(filepath); + + const p = parse(filepath); + if (p.ext === ".svg") { + return new Response(original); + } + + let path = `${w ? `w-${w}` : ""}${h ? `h-${h}` : ``}${ + fit ? `-${fit}` : "" + }`; + let file_name = dir( + `${g.datadir}/files/upload/thumb/${path}/${rpath}.webp` + ); + let file = Bun.file(file_name); + if (!(await file.exists())) { + await dirAsync(dirname(file_name)); + force = true; + } + + if (force) { + const img = sharp(await original.arrayBuffer()); + const arg: any = { fit: fit || "inside" }; + if (w) { + arg.width = w; + } + if (h) { + arg.height = h; + } + let out = img.resize(arg).webp({ quality: 75 }); + out = out.webp(); + + await Bun.write(file_name, new Uint8Array(await out.toBuffer())); + file = Bun.file(file_name); + } + + return new Response(file); + } + } + } catch (e: any) { + return new Response("ERROR:" + e.message, { status: 404 }); + } + + return res; + }, +}; diff --git a/internal/api/_kv.ts b/internal/api/_kv.ts new file mode 100644 index 0000000..167c911 --- /dev/null +++ b/internal/api/_kv.ts @@ -0,0 +1,60 @@ +import { BunSqliteKeyValue } from "pkgs/utils/kv"; +import { apiContext } from "utils/api-context"; +import { dir } from "utils/dir"; +import { g } from "utils/global"; + +export const _ = { + url: "/_kv", + raw: true, + async api(mode: "get" | "set" | "del", key: string, value?: any) { + const { req } = apiContext(this); + + if (!g.kv) { + g.kv = new BunSqliteKeyValue(dir(`${g.datadir}/db-kv.sqlite`)); + } + + try { + const parts = (await req.json()) as [string, string, any]; + switch (parts[0]) { + case "set": { + if (typeof parts[1] === "string") { + g.kv.set(parts[1], parts[2]); + + return new Response(JSON.stringify({ status: "ok" }), { + headers: { "content-type": "application/json" }, + }); + } + + return new Response( + JSON.stringify({ status: "failed", reason: "no key or body" }), + { + headers: { "content-type": "application/json" }, + } + ); + } + case "get": { + if (parts[2]) { + g.kv.set(parts[1], parts[2]); + } + + return new Response(JSON.stringify(g.kv.get(parts[1]) || null), { + headers: { "content-type": "application/json" }, + }); + } + case "del": { + if (parts[1]) { + g.kv.delete(parts[1]); + + return new Response(JSON.stringify({ status: "ok" }), { + headers: { "content-type": "application/json" }, + }); + } + } + } + } catch (e) {} + + return new Response(JSON.stringify({ status: "failed" }), { + headers: { "content-type": "application/json" }, + }); + }, +}; diff --git a/internal/api/_notif.ts b/internal/api/_notif.ts new file mode 100644 index 0000000..305c306 --- /dev/null +++ b/internal/api/_notif.ts @@ -0,0 +1,101 @@ +import { Database } from "bun:sqlite"; +import admin from "firebase-admin"; +import { listAsync } from "fs-jetpack"; +import { apiContext } from "utils/api-context"; + +import { dir } from "utils/dir"; +import { g } from "utils/global"; + +export const _ = { + url: "/_notif/:action/:token", + async api( + action: string, + data: + | { type: "register"; token: string; id: string } + | { type: "send"; id: string; body: string; title: string; data?: any } + ) { + const { req } = apiContext(this); + + if (action === "list") { + return await listAsync(dir("public")); + } + + if (!g.firebaseInit) { + g.firebaseInit = true; + + try { + g.firebase = admin.initializeApp({ + credential: admin.credential.cert(dir("public/firebase-admin.json")), + }); + g.notif = { + db: new Database(dir(`${g.datadir}/notif.sqlite`)), + }; + + g.notif.db.exec(` + CREATE TABLE IF NOT EXISTS notif ( + token TEXT PRIMARY KEY, + id TEXT NOT NULL + ); + `); + } catch (e) { + console.error(e); + } + } + + if (g.firebase) { + switch (action) { + case "register": + { + if (data && data.type === "register" && data.id) { + if (data.token) { + const q = g.notif.db.query( + `SELECT * FROM notif WHERE token = '${data.token}'` + ); + const result = q.all(); + if (result.length > 0) { + g.notif.db.exec( + `UPDATE notif SET id = '${data.id}' WHERE token = '${data.token}'` + ); + } else { + g.notif.db.exec( + `INSERT INTO notif VALUES ('${data.token}', '${data.id}')` + ); + } + + return { result: "OK" }; + } else { + return { error: "missing token" }; + } + } + } + break; + case "send": + { + if (data && data.type === "send") { + const q = g.notif.db.query<{ token: string }, any>( + `SELECT * FROM notif WHERE id = '${data.id}'` + ); + let result = q.all(); + for (const c of result) { + try { + await g.firebase.messaging().send({ + notification: { body: data.body, title: data.title }, + data: data.data, + token: c.token, + }); + } catch (e) { + console.error(e); + result = result.filter((v) => v.token !== c.token); + } + } + + return { result: "OK", totalDevice: result.length }; + } + } + break; + } + } + + return { error: "missing ./firebase-admin.json" }; + }, +}; diff --git a/internal/api/_prasi.ts b/internal/api/_prasi.ts new file mode 100644 index 0000000..fe1e111 --- /dev/null +++ b/internal/api/_prasi.ts @@ -0,0 +1,187 @@ +import { apiContext, createResponse } from "service-srv"; +import { SinglePage, g } from "utils/global"; +import { gzipAsync } from "utils/gzip"; +import { getContent } from "../server/prep-api-ts"; +import mime from "mime"; + +const cache = { + route: null as any, + comps: {} as Record, +}; +export const _ = { + url: "/_prasi/**", + async api() { + const { req, res } = apiContext(this); + res.setHeader("Access-Control-Allow-Origin", "*"); + res.setHeader("Access-Control-Allow-Headers", "content-type"); + const gz = g.deploy.content; + const parts = req.params._.split("/"); + + const action = { + _: () => { + res.send({ prasi: "v2" }); + }, + compress: async () => { + const last = parts.pop(); + if (last === "all") { + g.compress.mode = "all"; + } + if (last === "only-gz") { + g.compress.mode = "only-gz"; + } + if (last === "off") { + g.compress.mode = "off"; + } + }, + code: async () => { + if (gz) { + const path = parts.slice(1).join("/"); + if (gz.code.site[path]) { + const type = mime.getType(path); + if (type) res.setHeader("content-type", type); + res.send( + gz.code.site[path], + req.headers.get("accept-encoding") || "" + ); + } + } + }, + route: async () => { + if (gz) { + if (cache.route) return await responseCompressed(req, cache.route); + + let layout = null as null | SinglePage; + for (const l of gz.layouts) { + if (!layout) layout = l; + if (l.is_default_layout) layout = l; + } + + cache.route = JSON.stringify({ + site: { ...gz.site, api_url: (gz.site as any)?.config?.api_url }, + urls: gz.pages.map((e) => { + return { id: e.id, url: e.url }; + }), + layout: { + id: layout?.id, + root: layout?.content_tree, + }, + }); + + return await responseCompressed(req, cache.route); + } + }, + page: async () => { + const page = g.deploy.pages[parts[1]]; + if (page) { + const res = createResponse( + JSON.stringify({ + id: page.id, + root: page.content_tree, + url: page.url, + }), + { + cache_accept: req.headers.get("accept-encoding") || "", + high_compression: true, + } + ); + return res; + } + }, + pages: async () => { + const pages = []; + if (req.params.ids) { + for (const id of req.params.ids) { + const page = g.deploy.pages[id]; + if (page) { + pages.push({ + id: page.id, + root: page.content_tree, + url: page.url, + }); + } + } + } + + return await responseCompressed(req, JSON.stringify(pages)); + }, + comp: async () => { + const comps = {} as Record; + + const pending = new Set(); + if (req.params.ids) { + for (const id of req.params.ids) { + const comp = g.deploy.comps[id]; + if (comp) { + comps[id] = comp; + } else if (cache.comps[id]) { + comps[id] = cache.comps[id]; + } else { + pending.add(id); + } + } + } + + if (pending.size > 0) { + try { + const res = await fetch( + `https://prasi.avolut.com/prod/452e91b8-c474-4ed2-9c43-447ac8778aa8/_prasi/comp`, + { method: "POST", body: JSON.stringify({ ids: [...pending] }) } + ); + for (const [k, v] of Object.entries((await res.json()) as any)) { + cache.comps[k] = v; + comps[k] = v; + } + } catch (e) {} + } + + return createResponse(JSON.stringify(comps), { + cache_accept: req.headers.get("accept-encoding") || "", + high_compression: true, + }); + }, + "load.json": async () => { + res.setHeader("content-type", "application/json"); + res.send( + await getContent("load.json"), + req.headers.get("accept-encoding") || "" + ); + }, + "load.js": async () => { + res.setHeader("content-type", "text/javascript"); + + const url = req.query_parameters["url"] + ? JSON.stringify(req.query_parameters["url"]) + : "undefined"; + + if (req.query_parameters["dev"]) { + res.send( + await getContent("load.js.dev", url), + req.headers.get("accept-encoding") || "" + ); + } else { + res.send( + await getContent("load.js.prod", url), + req.headers.get("accept-encoding") || "" + ); + } + }, + }; + + const pathname: keyof typeof action = parts[0] as any; + const run = action[pathname]; + + if (run) { + return await run(); + } + }, +}; + +const responseCompressed = async (req: Request, body: string) => { + if (req.headers.get("accept-encoding")?.includes("gz")) { + return new Response(await gzipAsync(body), { + headers: { "content-encoding": "gzip" }, + }); + } + + return new Response(body); +}; diff --git a/internal/api/_proxy.ts b/internal/api/_proxy.ts new file mode 100644 index 0000000..2b79ff4 --- /dev/null +++ b/internal/api/_proxy.ts @@ -0,0 +1,47 @@ +import { apiContext } from "utils/api-context"; + +export const _ = { + url: "/_proxy/**", + raw: true, + async api() { + const { req } = apiContext(this); + + try { + const raw_url = decodeURIComponent(req.params["_"]); + const url = new URL(raw_url) as URL; + const body = await req.arrayBuffer(); + const headers = {} as Record; + req.headers.forEach((v, k) => { + if (k.startsWith("sec-")) return; + if (k.startsWith("connection")) return; + if (k.startsWith("dnt")) return; + if (k.startsWith("host")) return; + headers[k] = v; + }); + + const res = await fetch(url, { + method: req.method || "POST", + headers, + body, + }); + + if (res.headers.get("content-encoding")) { + res.headers.delete("content-encoding"); + } + + return res; + } catch (e: any) { + console.error(e); + new Response( + JSON.stringify({ + status: "failed", + reason: e.message, + }), + { + status: 403, + headers: { "content-type": "application/json" }, + } + ); + } + }, +}; diff --git a/internal/api/_upload.ts b/internal/api/_upload.ts new file mode 100644 index 0000000..0f473ec --- /dev/null +++ b/internal/api/_upload.ts @@ -0,0 +1,79 @@ +import mp from "@surfy/multipart-parser"; +import { dirAsync, existsAsync } from "fs-jetpack"; +import { format, parse, dirname } from "path"; +import { apiContext } from "utils/api-context"; +import { dir } from "utils/dir"; +import { g } from "utils/global"; + +export const _ = { + url: "/_upload", + raw: true, + async api(body: any) { + const { req } = apiContext(this); + const raw = await req.arrayBuffer(); + const parts = mp(Buffer.from(raw)) as Record< + string, + { fileName: string; mime: string; type: string; buffer: Buffer } + >; + + const result: string[] = []; + for (const [_, part] of Object.entries(parts)) { + result.push(await saveFile(req, part.fileName, part.buffer)); + } + + return new Response(JSON.stringify(result), { + headers: { "content-type": "application/json" }, + }); + }, +}; + +const saveFile = async ( + req: Request & { + params: any; + query_parameters: any; + }, + fname: string, + part: any +) => { + const d = new Date(); + let to: string = req.query_parameters["to"] || ""; + if (!to) { + to = `/upload/${d.getFullYear()}-${d.getMonth()}/${d.getDate()}/${d.getTime()}-${fname}`; + } else { + to = to + .split("/") + .map((e) => e.replace(/\.\./gi, "")) + .filter((e) => !!e) + .join("/"); + + to = to.endsWith("/") ? to + fname : to + "/" + fname; + } + to = to.toLowerCase(); + const pto = parse(to); + pto.name = pto.name.replace(/[\W_]+/gi, "-"); + to = format(pto); + + if (await existsAsync(dirname(to))) { + dirAsync(dirname(to)); + } + + while (await Bun.file(dir(`${g.datadir}/files/${to}`)).exists()) { + const p = parse(to); + const arr = p.name.split("-"); + if (arr.length > 1) { + if (parseInt(arr[arr.length - 1])) { + arr[arr.length - 1] = parseInt(arr[arr.length - 1]) + 1 + ""; + } else { + arr.push("1"); + } + } else { + arr.push("1"); + } + p.name = arr.filter((e) => e).join("-"); + p.base = `${p.name}${p.ext}`; + + to = format(p); + } + await Bun.write(dir(`${g.datadir}/files/${to}`), part); + return to; +}; diff --git a/internal/api/_zip.ts b/internal/api/_zip.ts new file mode 100644 index 0000000..22b6c9c --- /dev/null +++ b/internal/api/_zip.ts @@ -0,0 +1,144 @@ +import { $ } from "bun"; +import Database from "bun:sqlite"; +import { copyAsync } from "fs-jetpack"; +import mime from "mime"; +import { deploy } from "utils/deploy"; +import { dir } from "utils/dir"; +import { g, SinglePage } from "utils/global"; +import { getContent } from "../server/prep-api-ts"; + +export const _ = { + url: "/_zip", + raw: true, + async api() { + await $`rm bundle*`.nothrow().quiet().cwd(`${g.datadir}`); + await copyAsync( + dir(`pkgs/empty_bundle.sqlite`), + dir(`${g.datadir}/bundle.sqlite`) + ); + const db = new Database(dir(`${g.datadir}/bundle.sqlite`)); + + const ts = g.deploy.config.deploy.ts; + const add = ({ + path, + type, + content, + }: { + path: string; + type: string; + content: string | Buffer; + }) => { + if (path) { + const query = db.query( + "INSERT INTO files (path, type, content) VALUES ($path, $type, $content)" + ); + query.run({ + $path: path.startsWith(g.datadir) + ? path.substring(`${g.datadir}/bundle`.length) + : path, + $type: type, + $content: content, + }); + } + }; + + add({ path: "version", type: "", content: deploy.config.deploy.ts + "" }); + add({ path: "site_id", type: "", content: deploy.config.site_id + "" }); + add({ + path: "base_url", + type: "", + content: g.deploy.content?.site?.config?.api_url || "", + }); + const gz = g.deploy.content; + + if (gz) { + let layout = null as null | SinglePage; + for (const l of gz.layouts) { + if (!layout) layout = l; + if (l.is_default_layout) layout = l; + } + + let api_url = (gz.site as any)?.config?.api_url; + + add({ + path: "route", + type: "", + content: JSON.stringify({ + site: { + ...gz.site, + api_url, + }, + urls: gz.pages.map((e) => { + return { id: e.id, url: e.url }; + }), + layout: { + id: layout?.id, + root: layout?.content_tree, + }, + }), + }); + + add({ + path: "load-js", + type: "", + content: await getContent("load.js.prod", `"${api_url}"`), + }); + } + + for (const [directory, files] of Object.entries(g.deploy.content || {})) { + if (directory !== "code" && directory !== "site") { + for (const comp of Object.values(files) as any) { + let filepath = `${g.datadir}/bundle/${directory}/${comp.id}.json`; + + add({ + path: filepath, + type: mime.getType(filepath) || "text/plain", + content: JSON.stringify(comp), + }); + } + } else if (directory === "site") { + const filepath = `${g.datadir}/bundle/${directory}.json`; + add({ + path: filepath, + type: mime.getType(filepath) || "text/plain", + content: JSON.stringify(files), + }); + } else { + for (const [filename, content] of Object.entries(files)) { + let filepath = `${g.datadir}/bundle/${directory}/${filename}`; + + if (content instanceof Buffer || typeof content === "string") { + add({ + path: filepath, + type: mime.getType(filepath) || "text/plain", + content, + }); + } else { + for (const [k, v] of Object.entries(content || {})) { + filepath = `${g.datadir}/bundle/${directory}/${filename}/${k}`; + if (v instanceof Buffer || typeof v === "string") { + add({ + path: filepath, + type: mime.getType(filepath) || "text/plain", + content: v, + }); + } else { + add({ + path: filepath, + type: mime.getType(filepath) || "text/plain", + content: JSON.stringify(v), + }); + } + } + } + } + } + } + + await $`zip "bundle-${ts}.zip" bundle.sqlite` + .nothrow() + .quiet() + .cwd(`${g.datadir}`); + return new Response(Bun.file(`${g.datadir}/bundle-${ts}.zip`)); + }, +}; diff --git a/internal/server/server.ts b/internal/server/server.ts index 8e98a68..34f40c9 100644 --- a/internal/server/server.ts +++ b/internal/server/server.ts @@ -4,9 +4,12 @@ import type { ServerCtx } from "utils/server-ctx"; import { prasiContent } from "../content/content"; import { prasi_content_deploy } from "../content/content-deploy"; import { prasi_content_ipc } from "../content/content-ipc"; +import { fs } from "utils/fs"; startup("site", async () => { await config.init("site:site.json"); + fs.init(config.current!); + if (g.mode === "site") { g.prasi = g.ipc ? prasi_content_ipc : prasi_content_deploy; @@ -19,6 +22,18 @@ startup("site", async () => { const startSiteServer = async () => { if (g.mode === "site") { + let port = 0; + if (g.ipc) { + try { + const runtime = (await fs.read("site:runtime.json", "json")) as { + port: number; + }; + port = runtime.port; + } catch (e) {} + } else { + port = config.current?.port || 3000; + } + g.server = Bun.serve({ async fetch(req, server) { const content = prasiContent(); @@ -40,7 +55,13 @@ const startSiteServer = async () => { } }, websocket: { message(ws, message) {} }, - port: 0, + port, + reusePort: true, }); + + if (g.ipc && g.server.port) { + await g.ipc.backend?.server?.init?.({ port: g.server.port }); + await fs.write("site:runtime.json", { port: g.server.port }); + } } }; diff --git a/internal/supervisor.ts b/internal/supervisor.ts index 1058fd3..3df133a 100644 --- a/internal/supervisor.ts +++ b/internal/supervisor.ts @@ -7,12 +7,15 @@ import { prasi_content_deploy } from "./content/content-deploy"; import { ensureDBReady } from "./db/ensure"; import { ensureServerReady } from "./server/ensure"; import { startServer } from "./server/start"; +import { removeAsync } from "fs-jetpack"; const is_dev = process.argv.includes("--dev"); const is_ipc = process.argv.includes("--ipc"); startup("supervisor", async () => { console.log(`${c.green}Prasi Server:${c.esc} ${fs.path("site:")}`); await config.init("site:site.json"); + await removeAsync(fs.path(`site:runtime.json`)); + fs.init(config.current!); if (!is_ipc) { const site_id = config.get("site_id") as string; diff --git a/internal/utils/api-context.ts b/internal/utils/api-context.ts new file mode 100644 index 0000000..6a5f9c2 --- /dev/null +++ b/internal/utils/api-context.ts @@ -0,0 +1,111 @@ +import mime from "mime"; +import { binaryExtensions } from "./binary-ext"; + +const parseQueryParams = (ctx: any) => { + const pageHref = ctx.req.url; + const searchParams = new URLSearchParams( + pageHref.substring(pageHref.indexOf("?")) + ); + const result: any = {}; + searchParams.forEach((v, k) => { + result[k] = v; + }); + + return result as any; +}; +export const apiContext = (ctx: any) => { + ctx.req.params = ctx.params; + + if (ctx.params["_0"]) { + ctx.params["_"] = ctx.params["_0"]; + delete ctx.params["_0"]; + } + + ctx.req.query_parameters = parseQueryParams(ctx); + return { + req: ctx.req as Request & { params: any; query_parameters: any }, + res: { + ...ctx.res, + send: (body) => { + ctx.res = createResponse(body, { res: ctx.res }); + }, + sendStatus: (code: number) => { + ctx.res._status = code; + }, + setHeader: (key: string, value: string) => { + ctx.res.headers.append(key, value); + }, + } as Response & { + send: (body?: string | object) => void; + setHeader: (key: string, value: string) => void; + sendStatus: (code: number) => void; + }, + }; +}; + +(BigInt.prototype as any).toJSON = function (): string { + return `BigInt::` + this.toString(); +}; + +export const createResponse = ( + body: any, + opt?: { + headers?: any; + res?: any; + rewrite?: (arg: { + body: Bun.BodyInit; + headers: Headers | any; + }) => Bun.BodyInit; + } +) => { + const status = + typeof opt?.res?._status === "number" ? opt?.res?._status : undefined; + + const content_type = opt?.headers?.["content-type"]; + const is_binary = binaryExtensions.includes( + mime.getExtension(content_type) || "" + ); + const headers = { ...(opt?.headers || {}) } as Record; + + let pre_content = body; + if (opt?.rewrite) { + pre_content = opt.rewrite({ body: pre_content, headers }); + } + + let content: any = + typeof pre_content === "string" || is_binary + ? pre_content + : JSON.stringify(pre_content); + + let res = new Response( + content, + status + ? { + status, + } + : undefined + ); + + for (const [k, v] of Object.entries(headers)) { + res.headers.append(k, v); + } + const cur = opt?.res as Response; + if (cur) { + cur.headers.forEach((value, key) => { + res.headers.append(key, value); + }); + } + + if (typeof body === "object" && !res.headers.has("content-type")) { + res.headers.append("content-type", "application/json"); + } + + res.headers.append( + "Strict-Transport-Security", + "max-age=31536000; includeSubDomains; preload" + ); + + res.headers.append("X-Content-Type-Options", "nosniff"); + + return res; +}; diff --git a/internal/utils/binary-ext.ts b/internal/utils/binary-ext.ts new file mode 100644 index 0000000..377735e --- /dev/null +++ b/internal/utils/binary-ext.ts @@ -0,0 +1,263 @@ +export const binaryExtensions = [ + "3dm", + "3ds", + "3g2", + "3gp", + "7z", + "a", + "aac", + "adp", + "afdesign", + "afphoto", + "afpub", + "ai", + "aif", + "aiff", + "alz", + "ape", + "apk", + "appimage", + "ar", + "arj", + "asf", + "au", + "avi", + "bak", + "baml", + "bh", + "bin", + "bk", + "bmp", + "btif", + "bz2", + "bzip2", + "cab", + "caf", + "cgm", + "class", + "cmx", + "cpio", + "cr2", + "cur", + "dat", + "dcm", + "deb", + "dex", + "djvu", + "dll", + "dmg", + "dng", + "doc", + "docm", + "docx", + "dot", + "dotm", + "dra", + "DS_Store", + "dsk", + "dts", + "dtshd", + "dvb", + "dwg", + "dxf", + "ecelp4800", + "ecelp7470", + "ecelp9600", + "egg", + "eol", + "eot", + "epub", + "exe", + "f4v", + "fbs", + "fh", + "fla", + "flac", + "flatpak", + "fli", + "flv", + "fpx", + "fst", + "fvt", + "g3", + "gh", + "gif", + "graffle", + "gz", + "gzip", + "h261", + "h263", + "h264", + "icns", + "ico", + "ief", + "img", + "ipa", + "iso", + "jar", + "jpeg", + "jpg", + "jpgv", + "jpm", + "jxr", + "key", + "ktx", + "lha", + "lib", + "lvp", + "lz", + "lzh", + "lzma", + "lzo", + "m3u", + "m4a", + "m4v", + "mar", + "mdi", + "mht", + "mid", + "midi", + "mj2", + "mka", + "mkv", + "mmr", + "mng", + "mobi", + "mov", + "movie", + "mp3", + "mp4", + "mp4a", + "mpeg", + "mpg", + "mpga", + "mxu", + "nef", + "npx", + "numbers", + "nupkg", + "o", + "odp", + "ods", + "odt", + "oga", + "ogg", + "ogv", + "otf", + "ott", + "pages", + "pbm", + "pcx", + "pdb", + "pdf", + "pea", + "pgm", + "pic", + "png", + "pnm", + "pot", + "potm", + "potx", + "ppa", + "ppam", + "ppm", + "pps", + "ppsm", + "ppsx", + "ppt", + "pptm", + "pptx", + "psd", + "pya", + "pyc", + "pyo", + "pyv", + "qt", + "rar", + "ras", + "raw", + "resources", + "rgb", + "rip", + "rlc", + "rmf", + "rmvb", + "rpm", + "rtf", + "rz", + "s3m", + "s7z", + "scpt", + "sgi", + "shar", + "snap", + "sil", + "sketch", + "slk", + "smv", + "snk", + "so", + "stl", + "suo", + "sub", + "swf", + "tar", + "tbz", + "tbz2", + "tga", + "tgz", + "thmx", + "tif", + "tiff", + "tlz", + "ttc", + "ttf", + "txz", + "udf", + "uvh", + "uvi", + "uvm", + "uvp", + "uvs", + "uvu", + "viv", + "vob", + "war", + "wav", + "wax", + "wbmp", + "wdp", + "weba", + "webm", + "webp", + "whl", + "wim", + "wm", + "wma", + "wmv", + "wmx", + "woff", + "woff2", + "wrm", + "wvx", + "xbm", + "xif", + "xla", + "xlam", + "xls", + "xlsb", + "xlsm", + "xlsx", + "xlt", + "xltm", + "xltx", + "xm", + "xmind", + "xpi", + "xpm", + "xwd", + "xz", + "z", + "zip", + "zipx", +]; diff --git a/internal/utils/config.ts b/internal/utils/config.ts index 3e91933..7fff2b2 100644 --- a/internal/utils/config.ts +++ b/internal/utils/config.ts @@ -2,6 +2,7 @@ import { dirAsync } from "fs-jetpack"; import { fs } from "./fs"; import get from "lodash.get"; import set from "lodash.set"; +import { readdirSync } from "fs"; export const config = { async init(path: string) { @@ -10,8 +11,16 @@ export const config = { } const result = await fs.read(path, "json"); - this.current = result as typeof default_config; + if (!this.current) { + this.current = result as typeof default_config; + } this.file_path = path; + + const deploys = readdirSync(fs.path(`site:deploy/history`)); + this.current.deploy.history = deploys + .filter((e) => e.endsWith(".gz")) + .map((e) => parseInt(e.replace(".gz", ""))); + return result as typeof default_config; }, get(path: string) { @@ -32,7 +41,7 @@ const default_config = { db: { orm: "prisma" as "prisma" | "prasi", url: "" }, deploy: { current: 0, - history: [], + history: [] as number[], }, }; diff --git a/internal/utils/fs.ts b/internal/utils/fs.ts index 92d534c..f994c4c 100644 --- a/internal/utils/fs.ts +++ b/internal/utils/fs.ts @@ -2,6 +2,7 @@ import { mkdirSync, statSync } from "fs"; import { copyAsync } from "fs-jetpack"; import { g } from "./global"; import { dirname, join } from "path"; +import type { SiteConfig } from "./config"; const internal = Symbol("internal"); export const fs = { @@ -75,13 +76,15 @@ export const fs = { createPath: true, }); }, - init() { + init(config: SiteConfig) { this[internal].prefix.site = join(g.dir.root, "site"); + this[internal].prefix.upload = config.upload_path; this[internal].prefix.internal = join(process.cwd(), "internal"); }, [internal]: { prefix: { site: "", + upload: "", internal: "", }, }, diff --git a/internal/utils/global.ts b/internal/utils/global.ts index 15ad228..b984c73 100644 --- a/internal/utils/global.ts +++ b/internal/utils/global.ts @@ -1,6 +1,6 @@ import type { Server } from "bun"; import { join, resolve } from "path"; -import type { SiteConfig } from "./config"; +import { config, type SiteConfig } from "./config"; import { fs } from "./fs"; import type { PrasiSpawn, spawn } from "./spawn"; import type { prasi_content_ipc } from "../content/content-ipc"; @@ -75,7 +75,6 @@ export const startup = (mode: "supervisor" | "site", fn: () => void) => { } else { g.dir.root = join(process.cwd(), "..", ".."); } - fs.init(); fn(); }; diff --git a/internal/utils/parse-env.ts b/internal/utils/parse-env.ts new file mode 100644 index 0000000..7f74341 --- /dev/null +++ b/internal/utils/parse-env.ts @@ -0,0 +1,66 @@ +/** We don't normalize anything, so it is just strings and strings. */ +export type Data = Record; + +/** We typecast the value as a string so that it is compatible with envfiles. */ +export type Input = Record; + +// perhaps in the future we can use @bevry/json's toJSON and parseJSON and JSON.stringify to support more advanced types + +function removeQuotes(str: string) { + // Check if the string starts and ends with single or double quotes + if ( + (str.startsWith('"') && str.endsWith('"')) || + (str.startsWith("'") && str.endsWith("'")) + ) { + // Remove the quotes + return str.slice(1, -1); + } + // If the string is not wrapped in quotes, return it as is + return str; +} + +/** Parse an envfile string. */ +export function parseEnv(src: string): Data { + const result: Data = {}; + const lines = splitInLines(src); + for (const line of lines) { + const match = line.match(/^([^=:#]+?)[=:]((.|\n)*)/); + if (match) { + const key = match[1].trim(); + const value = removeQuotes(match[2].trim()); + result[key] = value; + } + } + return result; +} + +/** Turn an object into an envfile string. */ +export function genEnv(obj: Input): string { + let result = ""; + for (const [key, value] of Object.entries(obj)) { + if (key) { + const line = `${key}=${jsonValueToEnv(value)}`; + result += line + "\n"; + } + } + return result; +} + +function splitInLines(src: string): string[] { + return src + .replace(/("[\s\S]*?")/g, (_m, cg) => { + return cg.replace(/\n/g, "%%LINE-BREAK%%"); + }) + .split("\n") + .filter((i) => Boolean(i.trim())) + .map((i) => i.replace(/%%LINE-BREAK%%/g, "\n")); +} + +function jsonValueToEnv(value: any): string { + let processedValue = String(value); + processedValue = processedValue.replace(/\n/g, "\\n"); + processedValue = processedValue.includes("\\n") + ? `"${processedValue}"` + : processedValue; + return processedValue; +}