From 1f15dbe2bbfd831be3c09f9119b9324122c32635 Mon Sep 17 00:00:00 2001 From: Rizky Date: Mon, 29 Jul 2024 17:58:55 +0700 Subject: [PATCH] fixing hex --- bun.lockb | Bin 117484 -> 121581 bytes pkgs/api/_prasi.ts | 9 +- pkgs/package.json | 1 + pkgs/server/api-ctx.ts | 22 +++- pkgs/server/binary-ext.ts | 263 ++++++++++++++++++++++++++++++++++++++ pkgs/server/serve-web.ts | 3 +- pkgs/utils/cache.ts | 39 ++++-- pkgs/utils/deploy.ts | 23 +++- 8 files changed, 330 insertions(+), 30 deletions(-) create mode 100644 pkgs/server/binary-ext.ts diff --git a/bun.lockb b/bun.lockb index 446281478e404556fb224f8baaf473f865e1a88e..c394241bde4b2c5929af92be342ebea933a529a3 100755 GIT binary patch delta 8164 zcmeHMdwfjS_CI?h%;bSckVGP{dS?a zTaS1)8N?&5y4167QK}(SsEZ;O_ou(QTJCq9*(aIySFL;h{q=n2yT5y_wf8!E@3Z%w zv)8%5tkHwrjfw+0_(VU9_hs*$_S$u;-1$Mk{KB6vw)M8Jz7sg=G85>E zEYW+ZPKK8igsG!5^U`w+V<+W0#0o-FK`2@kCkX!FEY3Z_0gzp|d>iGokoSTE!7ISt z;&!#4DLp|DoS_*Bwg+ctrR5tl(}YZ8rg5xxd{!Lv4$$Wq@+W8~r4{z!vU3wb&_J() z>Us*`_yIo(emfB6NluVIX2_$S98sum&VSzAL$cRJpY(#*vxT9=I+_K?3A7P zz=Z3&53Jw&ujTzBoQ`fhVvcL}Q*TW;?m*AqUOepFX^D$k6Ng9F56T9jldDd13=cS~ zPU;eGEqw$jx6bA=#M(9+Qcj)iF{J!D>5HL)fPNR*iY4JXO_yPU&|Q|CJawv^Vd8>> zD9siq`$DM_OIqkO4+#yc3e8HzcVrQF<%hHGLpa8!_DeI@McA;(+8RRb`U6AUVpm(Fj4% ziwlz@Y*XrTyOE2Pbv7dfAwkY1Ava3S?Wxn*j}nBTmb%Er%ekF(I#r4w#E?!kC`BAF zDoXP<6cK2KAjbRZG>0Hz8Wb_=jH>Zy(Jm!QlaBdFqk`5QdvHU@0o)HUWdoUD5>&_paf3CJK7{M7nN(f4-kPa=DAyBH zR>x&)ZUOxut{=o2KoTR@7Z#HfgE7-7iNLU=WOoyzhSC6 z2MNun1>Da|+4LI)NcA2P;`cc(wJ=D<#_C#J==o6En)RIdW%sH6MADtbtj z$T0K4)Tw`N4j7g{H;4Mo84uPNFj{vU1*=yIj`X`X+w$?uf3|GNEVY2UK-Y|O0O z;`WCAxi?ByE}eVa&;3Ykzje>{{#aTSJY8zk?{(WHDy^Se@ufp^7xg&meQ&8--&>DH z(-tEt1Onqbr8jdihc2j71;A^P_Bi$jW^6mGa(c{jZ9#J<2aWgiv`-iZ0)uSr!my&OK;KQX}E^6=`n z7k#UjwlcT5@I`)cTo5$MzcliKP3}*QGtB=}yJYFi?O9&m?Qg&KYV7&S#G0`2-X7(< z)|zj*-Y)5JKKPrwd*A;Ua$r%?mgv9asnj+{OLuvh52;(MEtehJ&^bh~vu^NM7yGB% z@>4$S@3>;Yf^mCBHC*>Bqk-G--P5MTw+b=^`tOdOwsD{6xjN~<$o>WX$GWiE{E3@q z&Gq@~gM!ibJ#S_9mL02P$M)8aXF3L){@=dO-|RE$2j8VzoTgV6X+Iv{`kmIkv$NWj z&wu*<(y9aBJpSiDu2ejW+uXPB@@~^=5;sR1j`}1O**f-g*3thP!$zQ{7d!`R$1AI^ zR@>&KZe6;n&7#*2FMKCu?@tSC-RBqHt3DH5WgmTKnzq^T@VAE5#J1krJ7+=mN**jrY{jbgd||h+!MPn52lU&kUEK07$EO@R zvF46v>G<@yYxEAuey3-o)ja5$Q&aPH`on_*XH;I;<}>`=;<$~E7mH`kx(+nokJ{FJ z(}s}F{f1w5j&51zmRj39tcC8A4ue~tP_Mt$qg~sZNwG<1%1^f3~K~uuK3DPTzEEywG^u$^}FCGMxKjdZsPoNdhtg{4sdQ732A$=3~2{Zb4kxS ziUy>e#C=HpMW=7{Oe-3Z28f4|28wP~=u0*FQWe93#FLQ1t~!YRmj|*CvGB5;(Vmt#F}coDuRiP}X*LbIH{!E=yc&96fN# zX@&EN4(uk@+?j4m&)G&i7aC}eyWzm7Ob||QbZ(CTH#OJXnWGZVI;+jzKP>5nGW(e5 z{X?)_({7fz(K%A8?aZQZfEJ#=uw2Lrbkv{%^aeziBGa3Wjisj0ftR5jyeZdplqLr; zXHzF`qL*(EfG&ToBbOpCuH`y<(DZC;a z@#3%Za+p2BVy^2W#gmI@Zt{d0ArkrO%XO_G zAs9jdAK?w?s9Rn@4}h-z(t2_+fEV|HDiEO2h~YY43!NuUg z+|&l@FS#ieI=W+90O2c&a|!WWM-PDJ8Mz!#9(;`+3=v?HAPO8H&bYiz%?vI~o_ zZ46UW=_0ZQdgtE(&?fvLupamb_!#&E*Z^zUF+ zcn_E@P43D%u_Eb%uB?^Q7--Uf@jyCYlumYKdNxq<31wckuF$wky+T=qX(zIu0%ZXG zp_zeQ0KF!y0#*ZbKzkpcli=F`{iD7EybH_&<^xjzI@0C=89*kG1<+K~1Bt-57^OjA zS^-^w*BF9~ZIEdTPyoXdXbpG)KGGB&+th~6Y;=xW2xO!FWN-yI8GIc~E2o{*D~!cB z(h;8a4B8{K(#kLv+kw{WPLwhOvjWiNKg0~ ztQUZ@z&QZnH~|qhISz82<~ptm{pBZ48;%fBaW0&_Xy6+RQxJf@3A3U112+Sj05p3V zvWnVB_B}Jv)NlkG0DFMmAT)p)s6oM|U?<=+0<|jEH6WM_f)ZX=Lb*1Ks)WFz1mKlmZ@onc(JSF#ZZV;f z62(^{#`TqyIKL88uD2*(0+gt8z2$C%B{DzZ>03oMJ$ApLVkyCBT@V(S&JqFlQr$?2 zudU62rK|KjoplRUqWN(%Gb*dg%-dkmS$d?z``bUd_K&=l4=Z2^vv|`bNLdqDu!r)c z;I#T~+W~`5eZlY7SrFn9q)L=fjZBapPGGL8^aRPy$i}OD6Qx`u8>o7cD1Byx2PI_R zcgu@XkFwrP-~sKWk^VVJYCMs-hAKh+((q+DL);#(vtdC2T?5b^LHP{usQ=(mTO4}K zskekAOCwR&L-}y9cKfxAf2AgDs4p=lSt=&)%BO~2=f6F&X2izA`Vz{=2v7HQceft! zZe4FFOO~!tUFGvd=HO6=*?EKJ)t68{aO_%r=u-9HKVDUDd6q2sPeRLH(r?V8VWXu~ zln7P6Srm8}&(_RKc`T1@PyohO`6^L0Y{TSBH}jgQSWssS2`o=WOFK|kWt%E>&H%Sa zm2OPJD7Ql`7_T5$N(QTbSu@zR+#tU~6g{3ec!~GBH$U^m0tmvq zkWOT&$gpX~#^~+EiDXyRp6*S83(Ff$o`=JuXd~UYMhm z{c5vw@{QS9h72uz;xnYD<@NkO-hQpnjx*#=HD+lIxtXC|P~2qsJg8Of{<87~a_K)R zoMFtGQb4|6p&^#CLzRZSn(J3;hE%kbwH@&5p42}>Wmqu|A;DU>{H-ysGipPmL;1`v z=9Rnj5{H)l!pI%QzOqAkh<>Y$b;Dif^wnDlr@!lxw{p-cw31J$m-M^d%3YdX-Q`Pr zZas@%eR)_GOi!#BW#wR3`J*+Z%}>kAx2!kHp@vjb%)ER5&N9Tn{-IaA2}toDGoOl$ Jm8@^$e*unvMNR2M01{g5L{m2A>7{fqTHzPvi31CChQW+J>49HQ=5(jG;+$ z$6R-P)t`{2pr+<>#=;R>rg%1ZG_)NY1`Y>Dfw|Im${4diUjT=K-&6V!I0Skxn8wl$ zrm^o%b=PYJ)4dKIRJZ1auH89#Gr?9s#`xOcz|bn5L4k z(gj_M{nCBAZjbyX-EuRyJ<%5>Lw}qQbL`B`6h8$<&Dw_2url zABu)BM06(E4H0D`+~qJ-m5EB1!*~eKOA{_vmT};_QrSGloVxcmO67X#0ZJ~tRIYiv zBPh*vw?(N`FO|&qc%3K}lV|vRz6f9BFpa^Z%)&63=!>_T7D8dVn>pQt;b?`phAKXI zcr+upGfZ+WR1TD%=q%6hg`itvqB_-X994z+7p|#Urec()>y_Y}w*8kn4dw0tF@w(n zac%Kt!*dI|DwiBFHjTjI7=aCk)y2Zy8&Bt9&~eaY#Vh~+;h|`kpxP&beV`|+eu$}P z<#>Pw6p~fIo5{B+-TlNYCzsF>~s(fN9ZdZD{KL6f^C*Z|H5Z?(+ zmQX%16+4vnX0mo6!&{&ITE(BlO-~B^kWYQStvdE*dUOYsPfUydH%dQ&!}SW(@F*(M zogM>|{zKQA`kz1Ds82U4)&lnD(~YV&(hqG+nkw%&HSyk{6es;7Z`6v$q)M^xVlf{k z?p)NkugJTkag*4C+)sRoe7Gp;)wsXtL~a%jdo_`9(JWk-i+P|pd|4C!f-+wz=0T$3 ziY8vWWESs31q;(vO%(Q;Mg7%cK1%!<>H$>LwPGGBYOiVH-OFZi4k}E9UDw3?D`v6j zdNGd>r>|=~QpDZRc$C^oyaGPhsdp>^b?Iw5r;p)z1+gR^cV9K zQPGckx$Q4b_Rr+0U8Ya3hjcCa%)&)`$WXuYIK|Nk_}ZMVzdVY~5iAVg7mrBaHKDbsjxQ>?P-#Tn>s{oV0}jv@n87b#xVqyNH@y2IEQ zlFr^bhx>9$;*O@^IK^ ze^uT@<JCoV$_s<;0q7<( zWl%P{n5IGzu(kt~vXn>90lzTxl&w7ch+_C%pr;&JjV_uT2+@ti0Mp$OmaD4ALR*!W zr@T1mDZoe|UwQE;(<7tw6XnsoKGYq?o>AUd=nKmGsoM)+<6!(Sf@ zB4|q`$%A>^If0h^?@_A@pmp2S&8612iR8fbK3yX)1eSRQ7CrdZV7tg@N>KUH8o*{wTcr{!%`}UnZ>OC6S)5 zBeOTxoxE|p>$e8J&~8n(rZRTcDcjd_i=o#kkF4c3L%&nrBz?~*ACUgqDU%wYeP+vo z2ENSDGe`cefzLE_%$46XAl_aklh<)ey64-+&9u^uuJC=?2A*t9vDz^n&)1Q|nY&ic z3A?|4C{MAb>5=!!}s7trx%tDrFt3)Uv?v>D1FP~}Txv~ve=%18X_K`d2m%Tp25t}0a TUBoTj+6I2Fr2Cz-yddB|A@daO diff --git a/pkgs/api/_prasi.ts b/pkgs/api/_prasi.ts index 9ecff08..fe1e111 100644 --- a/pkgs/api/_prasi.ts +++ b/pkgs/api/_prasi.ts @@ -23,13 +23,13 @@ export const _ = { }, compress: async () => { const last = parts.pop(); - if (last === 'all') { + if (last === "all") { g.compress.mode = "all"; } - if (last === 'only-gz') { + if (last === "only-gz") { g.compress.mode = "only-gz"; } - if (last === 'off') { + if (last === "off") { g.compress.mode = "off"; } }, @@ -73,7 +73,7 @@ export const _ = { page: async () => { const page = g.deploy.pages[parts[1]]; if (page) { - return createResponse( + const res = createResponse( JSON.stringify({ id: page.id, root: page.content_tree, @@ -84,6 +84,7 @@ export const _ = { high_compression: true, } ); + return res; } }, pages: async () => { diff --git a/pkgs/package.json b/pkgs/package.json index fd748d1..51d9d30 100644 --- a/pkgs/package.json +++ b/pkgs/package.json @@ -9,6 +9,7 @@ "execa": "^8.0.1", "fs-jetpack": "^5.1.0", "mime": "^3.0.0", + "msgpackr": "^1.11.0", "parse-multipart-data": "^1.5.0", "pino": "^8.15.3", "pino-pretty": "^10.2.0", diff --git a/pkgs/server/api-ctx.ts b/pkgs/server/api-ctx.ts index 9f1cfa5..9da260c 100644 --- a/pkgs/server/api-ctx.ts +++ b/pkgs/server/api-ctx.ts @@ -1,6 +1,8 @@ import { simpleHash } from "utils/cache"; import { g } from "utils/global"; import { loadCachedBr } from "utils/br-load"; +import { binaryExtensions } from "./binary-ext"; +import mime from "mime"; const parseQueryParams = (ctx: any) => { const pageHref = ctx.req.url; @@ -60,7 +62,13 @@ export const createResponse = ( const status = typeof opt?.res?._status === "number" ? opt?.res?._status : undefined; - let content: any = typeof body === "string" ? body : JSON.stringify(body); + const content_type = opt?.headers?.["content-type"]; + const is_binary = binaryExtensions.includes( + mime.getExtension(content_type) || "" + ); + let content: any = + typeof body === "string" || is_binary ? body : JSON.stringify(body); + const headers = { ...(opt?.headers || {}) } as Record; if (opt?.cache_accept) { @@ -71,10 +79,6 @@ export const createResponse = ( ) { const content_hash = simpleHash(content); - if (!g.cache.br[content_hash]) { - loadCachedBr(content_hash, content); - } - if (g.cache.br[content_hash]) { cached = true; content = g.cache.br[content_hash]; @@ -85,11 +89,9 @@ export const createResponse = ( if (opt?.high_compression) { if (!cached && opt.cache_accept.toLowerCase().includes("gz")) { const content_hash = simpleHash(content); - if (!g.cache.gz[content_hash]) { g.cache.gz[content_hash] = Bun.gzipSync(content); } - if (g.cache.gz[content_hash]) { cached = true; content = g.cache.gz[content_hash]; @@ -111,6 +113,12 @@ export const createResponse = ( : undefined ); + if (opt?.headers?.["content-type"]?.includes("woff")) { + new Response(body, { + headers: { "content-type": headers["content-type"] }, + }); + } + for (const [k, v] of Object.entries(headers)) { res.headers.append(k, v); } diff --git a/pkgs/server/binary-ext.ts b/pkgs/server/binary-ext.ts new file mode 100644 index 0000000..377735e --- /dev/null +++ b/pkgs/server/binary-ext.ts @@ -0,0 +1,263 @@ +export const binaryExtensions = [ + "3dm", + "3ds", + "3g2", + "3gp", + "7z", + "a", + "aac", + "adp", + "afdesign", + "afphoto", + "afpub", + "ai", + "aif", + "aiff", + "alz", + "ape", + "apk", + "appimage", + "ar", + "arj", + "asf", + "au", + "avi", + "bak", + "baml", + "bh", + "bin", + "bk", + "bmp", + "btif", + "bz2", + "bzip2", + "cab", + "caf", + "cgm", + "class", + "cmx", + "cpio", + "cr2", + "cur", + "dat", + "dcm", + "deb", + "dex", + "djvu", + "dll", + "dmg", + "dng", + "doc", + "docm", + "docx", + "dot", + "dotm", + "dra", + "DS_Store", + "dsk", + "dts", + "dtshd", + "dvb", + "dwg", + "dxf", + "ecelp4800", + "ecelp7470", + "ecelp9600", + "egg", + "eol", + "eot", + "epub", + "exe", + "f4v", + "fbs", + "fh", + "fla", + "flac", + "flatpak", + "fli", + "flv", + "fpx", + "fst", + "fvt", + "g3", + "gh", + "gif", + "graffle", + "gz", + "gzip", + "h261", + "h263", + "h264", + "icns", + "ico", + "ief", + "img", + "ipa", + "iso", + "jar", + "jpeg", + "jpg", + "jpgv", + "jpm", + "jxr", + "key", + "ktx", + "lha", + "lib", + "lvp", + "lz", + "lzh", + "lzma", + "lzo", + "m3u", + "m4a", + "m4v", + "mar", + "mdi", + "mht", + "mid", + "midi", + "mj2", + "mka", + "mkv", + "mmr", + "mng", + "mobi", + "mov", + "movie", + "mp3", + "mp4", + "mp4a", + "mpeg", + "mpg", + "mpga", + "mxu", + "nef", + "npx", + "numbers", + "nupkg", + "o", + "odp", + "ods", + "odt", + "oga", + "ogg", + "ogv", + "otf", + "ott", + "pages", + "pbm", + "pcx", + "pdb", + "pdf", + "pea", + "pgm", + "pic", + "png", + "pnm", + "pot", + "potm", + "potx", + "ppa", + "ppam", + "ppm", + "pps", + "ppsm", + "ppsx", + "ppt", + "pptm", + "pptx", + "psd", + "pya", + "pyc", + "pyo", + "pyv", + "qt", + "rar", + "ras", + "raw", + "resources", + "rgb", + "rip", + "rlc", + "rmf", + "rmvb", + "rpm", + "rtf", + "rz", + "s3m", + "s7z", + "scpt", + "sgi", + "shar", + "snap", + "sil", + "sketch", + "slk", + "smv", + "snk", + "so", + "stl", + "suo", + "sub", + "swf", + "tar", + "tbz", + "tbz2", + "tga", + "tgz", + "thmx", + "tif", + "tiff", + "tlz", + "ttc", + "ttf", + "txz", + "udf", + "uvh", + "uvi", + "uvm", + "uvp", + "uvs", + "uvu", + "viv", + "vob", + "war", + "wav", + "wax", + "wbmp", + "wdp", + "weba", + "webm", + "webp", + "whl", + "wim", + "wm", + "wma", + "wmv", + "wmx", + "woff", + "woff2", + "wrm", + "wvx", + "xbm", + "xif", + "xla", + "xlam", + "xls", + "xlsb", + "xlsm", + "xlsx", + "xlt", + "xltm", + "xltx", + "xm", + "xmind", + "xpi", + "xpm", + "xwd", + "xz", + "z", + "zip", + "zipx", +]; diff --git a/pkgs/server/serve-web.ts b/pkgs/server/serve-web.ts index c72c636..9775808 100644 --- a/pkgs/server/serve-web.ts +++ b/pkgs/server/serve-web.ts @@ -7,9 +7,10 @@ export const serveWeb = async (arg: { cache_accept: string; }) => { const type = mime.getType(arg.pathname); + return createResponse(arg.content, { cache_accept: arg.cache_accept, - high_compression: true, + high_compression: false, headers: !type ? undefined : { "content-type": type }, }); }; diff --git a/pkgs/utils/cache.ts b/pkgs/utils/cache.ts index f964186..78d6a3d 100644 --- a/pkgs/utils/cache.ts +++ b/pkgs/utils/cache.ts @@ -1,15 +1,28 @@ -export const simpleHash = (str: string, seed = 0) => { - let h1 = 0xdeadbeef ^ seed, - h2 = 0x41c6ce57 ^ seed; - for (let i = 0, ch; i < str.length; i++) { - ch = str.charCodeAt(i); - h1 = Math.imul(h1 ^ ch, 2654435761); - h2 = Math.imul(h2 ^ ch, 1597334677); - } - h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507); - h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909); - h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507); - h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909); +import crypto from "crypto"; - return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString(); +function createHash(data: string, len: any) { + return crypto + .createHash("shake256", { outputLength: len }) + .update(data) + .digest("hex"); +} + +export const simpleHash = (str: string, seed = 0) => { + if (typeof str === "string") { + let h1 = 0xdeadbeef ^ seed, + h2 = 0x41c6ce57 ^ seed; + for (let i = 0, ch; i < str.length; i++) { + ch = str.charCodeAt(i); + h1 = Math.imul(h1 ^ ch, 2654435761); + h2 = Math.imul(h2 ^ ch, 1597334677); + } + h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507); + h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909); + h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507); + h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909); + + return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString(); + } + const hashed = createHash(str, 2); + return hashed; }; diff --git a/pkgs/utils/deploy.ts b/pkgs/utils/deploy.ts index ef666c4..339e0ed 100644 --- a/pkgs/utils/deploy.ts +++ b/pkgs/utils/deploy.ts @@ -12,6 +12,7 @@ import { gunzipAsync } from "./gzip"; import { createRouter } from "radix3"; import { prodIndex } from "./prod-index"; import { startBrCompress } from "./br-load"; +import { decode } from "msgpackr"; const decoder = new TextDecoder(); export const deploy = { @@ -28,15 +29,25 @@ export const deploy = { console.log(`Loading site: ${this.config.site_id} ${ts}`); try { - g.deploy.content = JSON.parse( - decoder.decode( + if (await Bun.file(`app/web/deploy/${ts}.mpack`).exists()) { + g.deploy.content = decode( await gunzipAsync( new Uint8Array( await Bun.file(dir(`app/web/deploy/${ts}.gz`)).arrayBuffer() ) ) - ) - ); + ); + } else { + g.deploy.content = JSON.parse( + decoder.decode( + await gunzipAsync( + new Uint8Array( + await Bun.file(dir(`app/web/deploy/${ts}.gz`)).arrayBuffer() + ) + ) + ) + ); + } if (g.deploy.content) { g.cache = { @@ -116,6 +127,7 @@ export const deploy = { } } catch (e) { console.log("Failed to load site", this.config.site_id); + console.error(e); } }, async run() { @@ -133,12 +145,13 @@ export const deploy = { `Downloading site deploy: ${this.config.site_id} [ts: ${this.config.deploy.ts}]` ); const res = await fetch( - `${base_url}/prod-zip/${this.config.site_id}?ts=${Date.now()}` + `${base_url}/prod-zip/${this.config.site_id}?ts=${Date.now()}&msgpack=1` ); const ts = Date.now(); const file = Bun.file(dir(`app/web/deploy/${ts}.gz`)); await Bun.write(file, await res.arrayBuffer()); + await Bun.write(dir(`app/web/deploy/${ts}.mpack`), "ok"); this.config.deploy.ts = ts + ""; await this.saveConfig();