diff --git a/bun.lockb b/bun.lockb index ba9c6de..4462814 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/package.json b/package.json index 9cb8d28..4ce5398 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,6 @@ }, "dependencies": { "brotli-wasm": "^2.0.1", - "bun-sqlite-key-value": "^1.4.5", "exit-hook": "^4.0.0", "firebase-admin": "^12.2.0", "prisma": "^5.17.0" diff --git a/pkgs/utils/global.ts b/pkgs/utils/global.ts index 6bc52c8..49c3976 100644 --- a/pkgs/utils/global.ts +++ b/pkgs/utils/global.ts @@ -3,10 +3,9 @@ import { Logger } from "pino"; import { RadixRouter } from "radix3"; import { PrismaClient } from "../../app/db/db"; -import admin from "firebase-admin"; import { Database } from "bun:sqlite"; -import { prodIndex } from "./prod-index"; -import { BunSqliteKeyValue } from "bun-sqlite-key-value"; +import admin from "firebase-admin"; +import { BunSqliteKeyValue } from "./kv"; type SingleRoute = { url: string; diff --git a/pkgs/utils/kv.ts b/pkgs/utils/kv.ts new file mode 100644 index 0000000..6283d1b --- /dev/null +++ b/pkgs/utils/kv.ts @@ -0,0 +1,430 @@ +import { Database, type Statement } from "bun:sqlite"; +import { serialize, deserialize } from "node:v8"; +import { dirname, resolve } from "node:path"; +import { existsSync, mkdirSync } from "node:fs"; + +export interface Item { + key: string; + value: T | undefined; +} + +interface Record { + key: string; + value: Buffer | null; + expires: number | null; +} + +interface Options { + readonly?: boolean; + create?: boolean; // Defaults to true + readwrite?: boolean; // Defaults to true + ttlMs?: number; // Default TTL milliseconds +} + +interface DbOptions extends Omit { + strict: boolean; +} + +const MIN_UTF8_CHAR: string = String.fromCodePoint(1); +const MAX_UTF8_CHAR: string = String.fromCodePoint(1_114_111); + +export class BunSqliteKeyValue { + db: Database; + ttlMs: number | undefined; + + private deleteExpiredStatement: Statement; + private deleteStatement: Statement; + private clearStatement: Statement; + private countStatement: Statement<{ count: number }>; + private setItemStatement: Statement; + private getItemStatement: Statement>; + private getAllItemsStatement: Statement; + private getItemsStartsWithStatement: Statement; + private getKeyStatement: Statement>; + private getAllKeysStatement: Statement>; + private getKeysStartsWithStatement: Statement>; + private countExpiringStatement: Statement<{ count: number }>; + private deleteExpiringStatement: Statement; + + // - `filename`: The full path of the SQLite database to open. + // Pass an empty string (`""`) or `":memory:"` or undefined for an in-memory database. + // - `options`: + // - ... + // - `ttlMs?: boolean`: Standard time period in milliseconds before + // an entry written to the DB becomes invalid. + constructor(filename?: string, options?: Options) { + // Parse options + const { ttlMs, ...otherOptions } = options ?? {}; + this.ttlMs = ttlMs; + const dbOptions: DbOptions = { + ...otherOptions, + strict: true, + readwrite: otherOptions?.readwrite ?? true, + create: otherOptions?.create ?? true, + }; + + // Create database directory + if ( + filename?.length && + filename.toLowerCase() !== ":memory:" && + dbOptions.create + ) { + const dbDir = dirname(resolve(filename)); + if (!existsSync(dbDir)) { + console.log(`The "${dbDir}" folder is created.`); + mkdirSync(dbDir, { recursive: true }); + } + } + + // Open database + this.db = new Database(filename, dbOptions); + this.db.run("PRAGMA journal_mode = WAL"); + + // Create table and indexes + this.db.run( + "CREATE TABLE IF NOT EXISTS items (key TEXT PRIMARY KEY, value BLOB, expires INT)" + ); + this.db.run( + "CREATE UNIQUE INDEX IF NOT EXISTS ix_items_key ON items (key)" + ); + this.db.run( + "CREATE INDEX IF NOT EXISTS ix_items_expires ON items (expires)" + ); + + // Prepare and cache statements + this.clearStatement = this.db.query("DELETE FROM items"); + this.deleteStatement = this.db.query("DELETE FROM items WHERE key = $key"); + this.deleteExpiredStatement = this.db.query( + "DELETE FROM items WHERE expires < $now" + ); + + this.setItemStatement = this.db.query( + "INSERT OR REPLACE INTO items (key, value, expires) VALUES ($key, $value, $expires)" + ); + this.countStatement = this.db.query("SELECT COUNT(*) AS count FROM items"); + + this.getAllItemsStatement = this.db.query( + "SELECT key, value, expires FROM items" + ); + this.getItemStatement = this.db.query( + "SELECT value, expires FROM items WHERE key = $key" + ); + this.getItemsStartsWithStatement = this.db.query( + "SELECT key, value, expires FROM items WHERE key = $key OR key >= $gte AND key < $lt" + ); + + this.getAllKeysStatement = this.db.query("SELECT key, expires FROM items"); + this.getKeyStatement = this.db.query( + "SELECT expires FROM items WHERE key = $key" + ); + this.getKeysStartsWithStatement = this.db.query( + "SELECT key, expires FROM items WHERE (key = $key OR key >= $gte AND key < $lt)" + ); + + this.countExpiringStatement = this.db.query( + "SELECT COUNT(*) as count FROM items WHERE expires IS NOT NULL" + ); + this.deleteExpiringStatement = this.db.query(` + DELETE FROM items + WHERE key IN ( + SELECT key FROM items + WHERE expires IS NOT NULL + ORDER BY expires ASC + LIMIT $limit + )`); + + // Delete expired items + this.deleteExpired(); + } + + // Delete all expired records + deleteExpired() { + this.deleteExpiredStatement.run({ now: Date.now() }); + } + + // Delete one or multiple items + delete(keyOrKeys?: string | string[]) { + if (typeof keyOrKeys === "string") { + // Delete one + this.deleteStatement.run({ key: keyOrKeys }); + } else if (keyOrKeys?.length) { + // Delete multiple items + this.db.transaction(() => { + keyOrKeys.forEach((key) => { + this.deleteStatement.run({ key }); + }); + })(); + } else { + // Delete all + this.clearStatement.run(); + } + } + + // Delete all items + clear() { + this.delete(); + } + + // Explicitly close database + // Removes .sqlite-shm and .sqlite-wal files + close() { + this.db.close(); + } + + // Returns the number of all items, including those that have already expired. + // First delete the expired items with `deleteExpired()` + // if you want to get the number of items that have not yet expired. + getCount(): number { + return (this.countStatement.get() as { count: number }).count; + } + + // Getter for getCount() + get length() { + return this.getCount(); + } + + // @param ttlMs: + // Time to live in milliseconds. + // Set ttlMs to 0 if you explicitly want to disable expiration. + set(key: string, value: T, ttlMs?: number) { + let expires: number | undefined; + ttlMs = ttlMs ?? this.ttlMs; + if (ttlMs !== undefined && ttlMs > 0) { + expires = Date.now() + ttlMs; + } + this.setItemStatement.run({ key, value: serialize(value), expires }); + } + + // Alias for `set` + setValue = this.set; + put = this.set; + + // Adds a large number of entries to the database and takes only + // a small fraction of the time that `set()` would take individually. + setItems(items: { key: string; value: T; ttlMs?: number }[]) { + this.db.transaction(() => { + items.forEach(({ key, value, ttlMs }) => { + this.set(key, value, ttlMs); + }); + })(); + } + + // Get one value + get(key: string): T | undefined { + const record = this.getItemStatement.get({ key }); + if (!record) return; + const { value, expires } = record; + if (expires) { + if (expires < Date.now()) { + this.delete(key); + return; + } + } + return value ? (deserialize(value) as T) : undefined; + } + + // Alias for `get` + getValue = this.get; + + // Get one item (key, value) + getItem(key: string): Item | undefined { + return { + key, + value: this.get(key), + }; + } + + // Get multiple items (key-value array) + getItems( + startsWithOrKeys?: string | string[] + ): Item[] | undefined { + let records: Record[]; + if (startsWithOrKeys && typeof startsWithOrKeys === "string") { + // Filtered items (startsWith) + // key = "addresses:" + // gte = key + MIN_UTF8_CHAR + // "addresses:aaa" + // "addresses:xxx" + // lt = key + MAX_UTF8_CHAR + const key: string = startsWithOrKeys; + const gte: string = key + MIN_UTF8_CHAR; + const lt: string = key + MAX_UTF8_CHAR; + records = this.getItemsStartsWithStatement.all({ key, gte, lt }); + } else if (startsWithOrKeys) { + // Filtered items (array with keys) + records = this.db.transaction(() => { + return (startsWithOrKeys as string[]).map((key: string) => { + const record = this.getItemStatement.get({ key }); + return { ...record, key }; + }); + })(); + } else { + // All items + records = this.getAllItemsStatement.all(); + } + if (!records.length) return; + const now = Date.now(); + const result: Item[] = []; + for (const record of records) { + const { key, value, expires } = record; + if (expires && expires < now) { + this.delete(key); + } else { + result.push({ + key, + value: value ? (deserialize(value) as T) : undefined, + }); + } + } + if (result.length) { + return result; + } + } + + // Alias for getItems + getItemsArray = this.getItems; + + // Get multiple values as array + getValues( + startsWithOrKeys?: string | string[] + ): (T | undefined)[] | undefined { + return this.getItems(startsWithOrKeys)?.map((result) => result.value); + } + + // Alias for getValues + getValuesArray = this.getValues; + + // Get multiple items as object + getItemsObject( + startsWithOrKeys?: string | string[] + ): { [key: string]: T | undefined } | undefined { + const items = this.getItems(startsWithOrKeys); + if (!items) return; + return Object.fromEntries( + items.map((item) => [item.key, item.value as T | undefined]) + ); + } + + // Get multiple items as Map() + getItemsMap( + startsWithOrKeys?: string | string[] + ): Map | undefined { + const items = this.getItems(startsWithOrKeys); + if (!items) return; + return new Map( + items.map((item) => [item.key, item.value as T | undefined]) + ); + } + + // Get multiple values as Set() + getValuesSet( + startsWithOrKeys?: string | string[] + ): Set | undefined { + const values = this.getValues(startsWithOrKeys); + if (!values) return; + return new Set(values); + } + + // Checks if key exists + has(key: string): boolean { + const record = this.getKeyStatement.get({ key }); + if (!record) return false; + if (record.expires) { + if (record.expires < Date.now()) { + this.delete(key); + return false; + } + } + return true; + } + + // Get multiple keys as array + getKeys(startsWithOrKeys?: string | string[]): string[] | undefined { + let records: (Omit | undefined)[]; + if (startsWithOrKeys && typeof startsWithOrKeys === "string") { + const key: string = startsWithOrKeys; + const gte: string = key + MIN_UTF8_CHAR; + const lt: string = key + MAX_UTF8_CHAR; + records = this.getKeysStartsWithStatement.all({ key, gte, lt }); + } else if (startsWithOrKeys) { + // Filtered items (array with keys) + records = this.db.transaction(() => { + return (startsWithOrKeys as string[]).map((key: string) => { + const record = this.getKeyStatement.get({ key }); + return record ? { ...record, key } : undefined; + }); + })(); + } else { + // All items + records = this.getAllKeysStatement.all(); + } + if (!records?.length) return; + const now = Date.now(); + const result: string[] = []; + for (const record of records) { + if (!record) continue; + const { key, expires } = record; + if (expires && expires < now) { + this.delete(key); + } else { + result.push(key); + } + } + if (result.length) { + return result; + } + } + + getExpiringItemsCount() { + return this.countExpiringStatement.get()!.count; + } + + // If there are more expiring items in the database than `maxExpiringItemsInDb`, + // the oldest items are deleted until there are only `maxExpiringItemsInDb` items with + // an expiration date in the database. + deleteOldExpiringItems(maxExpiringItemsInDb: number) { + const count = this.getExpiringItemsCount(); + if (count <= maxExpiringItemsInDb) return; + + const limit = count - maxExpiringItemsInDb; + this.deleteExpiringStatement.run({ limit }); + } + + // Alias for deleteOldExpiringItems + deleteOldestExpiringItems = this.deleteOldExpiringItems; + + // Proxy + getDataObject(): { [key: string]: any } { + const self = this; + return new Proxy( + {}, + { + get(target, property: string, receiver) { + if (property === "length") { + return self.length; + } else { + return self.get(property); + } + }, + + set(target, property: string, value: any) { + self.set(property, value); + return true; + }, + + has(target, property: string) { + return self.has(property); + }, + + deleteProperty(target: {}, property: string) { + self.delete(property); + return true; + }, + } + ); + } + + get dataObject() { + return this.getDataObject(); + } +}