From 5f8f581b63fba8bbd6cae572a2deeb8e4364f194 Mon Sep 17 00:00:00 2001 From: riz Date: Wed, 19 Nov 2025 10:12:33 +0000 Subject: [PATCH] Replace msgpack encoding with actual zip file creation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Completely rewrite /prod-zip endpoint to create real ZIP archives - Add archiver dependency for proper ZIP file creation - Create organized file structure in ZIP: * metadata.json - site configuration, layouts, pages, components * public/ - all public files * server/ - server build files * site/ - site build files (limited to 500 files to prevent enormous archives) * core/ - core application files * site-files.json - listing of included/skipped site files Benefits: - No more msgpack buffer overflow issues - Creates actual usable ZIP files that can be extracted - Much more practical for developers to work with - Includes file structure and metadata - Handles large sites by limiting build file inclusion - Proper ZIP compression with archive headers - Returns with appropriate Content-Type and Content-Disposition headers This transforms the endpoint from returning complex binary data to providing actual site exports. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- app/srv/api/prod-zip.ts | 211 +++++++++++++++++++++++----------------- package.json | 1 + 2 files changed, 123 insertions(+), 89 deletions(-) diff --git a/app/srv/api/prod-zip.ts b/app/srv/api/prod-zip.ts index 85ce705f..3bb051f5 100644 --- a/app/srv/api/prod-zip.ts +++ b/app/srv/api/prod-zip.ts @@ -1,77 +1,115 @@ import { apiContext } from "service-srv"; import fs from "fs"; import path from "path"; -import { gzipAsync } from "../ws/sync/entity/zlib"; import { validate } from "uuid"; import { dir } from "dir"; import { existsAsync, readAsync, exists } from "fs-jetpack"; import { code } from "../ws/sync/code/code"; -import { encode, Packr } from "msgpackr"; import { binaryExtensions } from "../util/binary-ext"; -// Create a custom Packr instance with larger buffer limits -const largePackr = new Packr({ - // Configure for large data handling - useRecords: false, - // Set reasonable chunk size for encoding - maxStructureDepth: 64, - // Additional options for large data -}); +// Import archiver for zip creation +const archiver = require('archiver'); -function encodeLargeData(data: any): Uint8Array { - try { - // Try standard encoding first - return encode(data); - } catch (e) { - // If that fails, try with our custom packr - console.warn(`Standard msgpack failed for large data, using custom packr: ${e.message}`); - return largePackr.encode(data); - } -} +// Create a zip archive containing site files and metadata +async function createSiteZip(site_id: string, siteData: any): Promise { + return new Promise((resolve, reject) => { + const archive = archiver('zip', { + zlib: { level: 9 } // Maximum compression + }); -// Process file contents separately to avoid buffer overflow - very restrictive -function processFileContents(fileData: Record, mode: "string" | "binary"): Record { - const result: Record = {}; - let processedCount = 0; - let totalSize = 0; - const maxSizeLimit = 50 * 1024 * 1024; // 50MB total limit per section - const maxFileCount = 1000; // Strict limit on number of files - const maxFileSize = 1 * 1024 * 1024; // 1MB per file limit + const chunks: Buffer[] = []; + archive.on('data', (chunk: Buffer) => chunks.push(chunk)); + archive.on('end', () => resolve(Buffer.concat(chunks))); + archive.on('error', reject); - console.log(`Processing file contents with strict limits: max ${maxFileCount} files, ${maxFileSize} per file, ${maxSizeLimit} total`); + // Create a temporary directory structure in memory + console.log(`Creating zip archive for site: ${site_id}`); - for (const [key, content] of Object.entries(fileData)) { - if (processedCount >= maxFileCount) { - console.warn(`Reached maximum file count ${maxFileCount}, stopping processing`); - break; + try { + // Add metadata as JSON file + const metadata = { + site: siteData.site, + layouts: siteData.layouts, + pages: siteData.pages, + components: siteData.comps, + created_at: new Date().toISOString(), + site_id: site_id + }; + + archive.append(JSON.stringify(metadata, null, 2), { name: 'metadata.json' }); + + // Add public files + if (siteData.public) { + console.log(`Adding ${Object.keys(siteData.public).length} public files...`); + for (const [filePath, content] of Object.entries(siteData.public)) { + if (typeof content === 'string') { + archive.append(content, { name: `public/${filePath}` }); + } else { + archive.append(Buffer.from(content), { name: `public/${filePath}` }); + } + } + } + + // Add server build files + if (siteData.code?.server) { + console.log(`Adding ${Object.keys(siteData.code.server).length} server files...`); + for (const [filePath, content] of Object.entries(siteData.code.server)) { + if (typeof content === 'string') { + archive.append(content, { name: `server/${filePath}` }); + } else { + archive.append(Buffer.from(content), { name: `server/${filePath}` }); + } + } + } + + // Add site build files (limiting to prevent zip from becoming too large) + if (siteData.code?.site) { + const siteFiles = Object.entries(siteData.code.site); + const maxFiles = 500; // Limit site files to prevent zip from being enormous + let fileCount = 0; + + console.log(`Adding up to ${maxFiles} site files...`); + for (const [filePath, content] of siteFiles) { + if (fileCount >= maxFiles) { + console.log(`Reached site file limit (${maxFiles}), stopping...`); + break; + } + + if (typeof content === 'string') { + archive.append(content, { name: `site/${filePath}` }); + } else { + archive.append(Buffer.from(content), { name: `site/${filePath}` }); + } + fileCount++; + } + + // Add a file listing what was included and what was skipped + const fileListing = { + included: fileCount, + total: siteFiles.length, + skipped: siteFiles.length - fileCount, + files: siteFiles.slice(0, maxFiles).map(([path]) => path) + }; + archive.append(JSON.stringify(fileListing, null, 2), { name: 'site-files.json' }); + } + + // Add core files + if (siteData.code?.core) { + console.log(`Adding ${Object.keys(siteData.code.core).length} core files...`); + for (const [filePath, content] of Object.entries(siteData.code.core)) { + if (typeof content === 'string') { + archive.append(content, { name: `core/${filePath}` }); + } else { + archive.append(Buffer.from(content), { name: `core/${filePath}` }); + } + } + } + + archive.finalize(); + } catch (error) { + reject(error); } - - const contentSize = typeof content === 'string' ? content.length : content.byteLength; - - // Check if adding this file would exceed total size limit - if (totalSize + contentSize > maxSizeLimit) { - console.warn(`Would exceed size limit, skipping file ${key} (${contentSize} bytes)`); - break; - } - - // Skip extremely large files - if (contentSize > maxFileSize) { - console.warn(`Skipping large file ${key} (${contentSize} bytes) - exceeds ${maxFileSize} limit`); - result[key] = mode === "binary" ? new Uint8Array(0) : ""; // Empty content as placeholder - } else { - result[key] = content; - totalSize += contentSize; - } - - processedCount++; - - if (processedCount % 100 === 0) { - console.log(`Processed ${processedCount} files, total size: ${(totalSize / 1024 / 1024).toFixed(2)}MB`); - } - } - - console.log(`Completed processing ${processedCount} files (${(totalSize / 1024 / 1024).toFixed(2)}MB total), skipped ${Object.keys(fileData).length - processedCount} files`); - return result; + }); } // Manual minimal msgpack encoder as ultimate fallback @@ -294,28 +332,16 @@ export const _ = { async api(site_id: string) { const { req, res } = apiContext(this); - let is_msgpack = req.query_parameters["msgpack"]; - - // Add timeout handling for large file operations + // Add timeout handling for zip creation const timeoutPromise = new Promise((_, reject) => { - setTimeout(() => reject(new Error('Request timeout')), 230000); // 230 seconds (less than server timeout of 240) + setTimeout(() => reject(new Error('Request timeout')), 230000); // 230 seconds }); const zipPromise = (async () => { if (validate(site_id)) { - const mode = is_msgpack ? "binary" : "string"; + console.log(`Starting zip creation for site: ${site_id}`); - // Check public directory size first to estimate total size - const public_data = readDirectoryRecursively( - mode, - code.path(site_id, "site", "src", "public") - ); - - // Estimate if this will be too large for msgpack - const estimatedSize = Object.keys(public_data).length * 1000; // rough estimate - if (estimatedSize > 100 * 1024 * 1024) { // 100MB estimate - console.warn(`Large site detected for ${site_id}, estimated size: ${estimatedSize} bytes`); - } + // Fetch all the site data const result = { layouts: await _db.page.findMany({ where: { @@ -355,7 +381,10 @@ export const _ = { }, select: { id: true, content_tree: true }, }), - public: public_data, + public: readDirectoryRecursively( + "binary", + code.path(site_id, "site", "src", "public") + ), site: await _db.site.findFirst({ where: { id: site_id }, select: { @@ -368,28 +397,32 @@ export const _ = { }), code: { server: readDirectoryRecursively( - mode, + "binary", code.path(site_id, "server", "build") ), site: readDirectoryRecursively( - mode, + "binary", code.path(site_id, "site", "build") ), - core: readDirectoryRecursively(mode, dir.path(`/app/srv/core`)), + core: readDirectoryRecursively("binary", dir.path(`/app/srv/core`)), }, }; - let dataToCompress: Uint8Array; + // Create the zip file + console.log(`Creating zip archive with ${Object.keys(result.public || {}).length} public files`); + const zipBuffer = await createSiteZip(site_id, result); - // Use optimized msgpack encoding for large data - if (mode === "binary") { - dataToCompress = encodeVeryLargeData(result); - } else { - // For string mode, still use JSON as it's more appropriate for text - dataToCompress = new TextEncoder().encode(JSON.stringify(result)); - } + console.log(`Zip created successfully: ${zipBuffer.length} bytes`); - return await gzipAsync(Buffer.from(dataToCompress)); + // Return the zip file with appropriate headers + return new Response(zipBuffer, { + status: 200, + headers: { + 'Content-Type': 'application/zip', + 'Content-Disposition': `attachment; filename="site-${site_id}-${Date.now()}.zip"`, + 'Cache-Control': 'no-cache', + }, + }); } return new Response("NOT FOUND", { status: 403 }); })(); diff --git a/package.json b/package.json index dbd6b5ab..b1e58490 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "pkgs/*" ], "dependencies": { + "archiver": "^6.0.1", "brotli-wasm": "^3.0.1", "caniuse-lite": "^1.0.30001688", "fdir": "^6.3.0",