Diagnosing performance

import { monitorEventLoopDelay } from "node:perf_hooks";
import { createHttpServer, createNodeHttpAdapter, createResponseContext, createRouter } from "@zudojs/http";

const tasks = Array.from({ length: 10_000 }, (_, i) => ({
  id: `t-${i}`, ownerId: "u-ada", title: `Task number ${i}`, done: i % 3 === 0, createdAt: "2026-09-01T09:00:00.000Z",
}));
const router = createRouter();
router.get("/health", () => createResponseContext().json({ status: "ok" }));
router.get("/tasks/all", () => createResponseContext().json(tasks));
router.get("/tasks/page", () => createResponseContext().json(tasks.slice(0, 50)));
const server = createHttpServer({
  adapter: createNodeHttpAdapter({ host: "127.0.0.1", port: 0, keepAliveTimeout: 60_000 }),
  handler: async (request) => (await router.dispatch(request)).response,
});
await server.start();
const base = `http://127.0.0.1:${server.address?.port}`;

async function timed(path: string): Promise<number> {
  const start = performance.now();
  await (await fetch(base + path)).arrayBuffer();
  return performance.now() - start;
}
const median = (values: number[]) => values.sort((a, b) => a - b)[Math.floor(values.length / 2)]!;

async function scenario(path: string) {
  const delay = monitorEventLoopDelay({ resolution: 5 });
  delay.enable();
  const health: number[] = [];
  let loading = true;
  const probe = (async () => {
    while (loading) {
      health.push(await timed("/health"));
      await new Promise((resolve) => setTimeout(resolve, 5));
    }
  })();
  const latencies: number[] = [];
  let sent = 0;
  await Promise.all(Array.from({ length: 10 }, async () => {
    while (sent++ < 100) latencies.push(await timed(path));
  }));
  loading = false;
  await probe;
  delay.disable();
  return { median: median(latencies), healthMedian: median(health), delayP90: delay.percentile(90) / 1e6 };
}

await scenario("/tasks/page");
const all = await scenario("/tasks/all");
const page = await scenario("/tasks/page");
console.log("full list median at least 3x the page median:", all.median > 3 * page.median);
console.log("/health median at least 2x slower while the full list is served:", all.healthMedian > 2 * page.healthMedian);
console.log("event-loop delay (p90) at least 2x higher:", all.delayP90 > 2 * page.delayP90);
await server.stop();

full list median at least 3x the page median: true
/health median at least 2x slower while the full list is served: true
event-loop delay (p90) at least 2x higher: true

import { monitorEventLoopDelay } from "node:perf_hooks";

// How late the event loop runs, every 10 s: near 0 when idle, high when code hogs the CPU.
// The histogram measures the time between samples, so subtract the sampling interval.
const SAMPLE_MS = 10;
const loopDelay = monitorEventLoopDelay({ resolution: SAMPLE_MS });
loopDelay.enable();
setInterval(() => {
  const late = (ns: number) => Math.max(0, Math.round(ns / 1e6) - SAMPLE_MS);
  console.log(`event loop delay: p50=${late(loopDelay.percentile(50))}ms p99=${late(loopDelay.percentile(99))}ms max=${late(loopDelay.max)}ms`);
  loopDelay.reset();
}, 10_000).unref();

// Usage: node top-functions.mjs profiles/CPU.….cpuprofile
import { readFileSync } from "node:fs";
import { basename } from "node:path";

const profile = JSON.parse(readFileSync(process.argv[2], "utf8"));
const byId = new Map(profile.nodes.map((node) => [node.id, node]));
const selfTime = new Map();
profile.samples.forEach((id, i) => {
  const { functionName, url, lineNumber } = byId.get(id).callFrame;
  const name = `${functionName || "(anonymous)"} ${url ? `${basename(url)}:${lineNumber + 1}` : ""}`.trim();
  selfTime.set(name, (selfTime.get(name) ?? 0) + (profile.timeDeltas[i] ?? 0));
});
const total = [...selfTime.values()].reduce((a, b) => a + b, 0);
for (const [name, time] of [...selfTime].sort((a, b) => b[1] - a[1]).slice(0, 8)) {
  console.log(`${((time / total) * 100).toFixed(1).padStart(5)}%  ${name}`);
}

const tasks = Array.from({ length: 10_000 }, (_, i) => ({
  id: crypto.randomUUID(), name: `Task number ${i}`, createdAt: "2026-09-01T09:00:00.000Z", updatedAt: "2026-09-01T09:00:00.000Z",
}));
const bytes = (value: unknown) => new TextEncoder().encode(JSON.stringify(value)).length;

function msFor(value: unknown, runs: number): number {
  const start = performance.now();
  for (let run = 0; run < runs; run++) JSON.stringify(value);
  return (performance.now() - start) / runs;
}
msFor(tasks, 5);
const full = msFor(tasks, 20);
const page = msFor({ items: tasks.slice(0, 50), next: tasks[49]!.id }, 2_000);

console.log("full list bytes:", bytes(tasks));
console.log("page of 50 bytes:", bytes({ items: tasks.slice(0, 50), next: tasks[49]!.id }));
console.log("stringify of the full list at least 50x slower:", full > 50 * page);

full list bytes: 1498891
page of 50 bytes: 7447
stringify of the full list at least 50x slower: true

/** Query of `GET /api/v1/tasks`: at most 100 per page, 50 by default. */
export const ListTasksQuerySchema = schema.object({
  limit: schema.coerce.number().int().min(1).max(100).default(50),
  after: schema.string().uuid().optional(),
});

public readonly list = async (ctx: HttpRouterContext): Promise<HttpResponseContext> => {
  const query = ListTasksQuerySchema.safeParse(ctx.query);
  if (!query.success) return validationFailed(query.issues);
  const items = await this.service.list(query.data.limit, query.data.after);
  const next = items.length === query.data.limit ? items.at(-1)?.id : undefined;
  return json(200, { items, next });
};

import { createRateLimiter } from "@zudojs/security";

function lastThousandMs(max: number, checks: number): number {
  const limiter = createRateLimiter({ windowMs: 60_000, max });
  for (let i = 0; i < checks - 1_000; i++) limiter.check({ ip: "203.0.113.7" });
  const start = performance.now();
  for (let i = 0; i < 1_000; i++) limiter.check({ ip: "203.0.113.7" });
  const ms = performance.now() - start;
  limiter.destroy();
  return ms;
}

lastThousandMs(300, 8_000);
const production = lastThousandMs(300, 8_000);
const loadTest = lastThousandMs(1_000_000, 8_000);
const productionLater = lastThousandMs(300, 16_000);
console.log("after 8,000 checks from one client, a limit of 1,000,000 costs at least 5x a limit of 300:", loadTest > 5 * production);
console.log("with a limit of 300, twice as many earlier checks cost no more than 2x:", productionLater < 2 * production);

after 8,000 checks from one client, a limit of 1,000,000 costs at least 5x a limit of 300: true
with a limit of 300, twice as many earlier checks cost no more than 2x: true

import { HttpMiddlewarePipeline, createRequestContext, createResponseContext, type HttpMiddleware } from "@zudojs/http";

const tasks = Array.from({ length: 2_000 }, (_, i) => ({ id: i, title: `Task number ${i}`, done: false }));
const handler: HttpMiddleware = async () => createResponseContext().json(tasks);
const logged: string[] = [];
const logFullBody: HttpMiddleware = async (_context, next) => {
  const response = await next();
  const body: unknown = JSON.parse(String(response.body));
  logged.push(JSON.stringify({ status: response.status, body }));
  logged.length = 0;
  return response;
};

const request = createRequestContext({ method: "GET", url: "http://localhost/tasks" });
async function msFor50(pipeline: HttpMiddlewarePipeline): Promise<number> {
  const start = performance.now();
  for (let i = 0; i < 50; i++) await pipeline.execute(request, createResponseContext());
  return performance.now() - start;
}

const plain = new HttpMiddlewarePipeline({ middlewares: [handler] });
const withLogging = new HttpMiddlewarePipeline({ middlewares: [logFullBody, handler] });
const best = { plain: Infinity, withLogging: Infinity };
for (let round = 0; round < 6; round++) {
  best.plain = Math.min(best.plain, await msFor50(plain));
  best.withLogging = Math.min(best.withLogging, await msFor50(withLogging));
}
console.log("logging the full body at least 1.5x the cost of the request:", best.withLogging > 1.5 * best.plain);

logging the full body at least 1.5x the cost of the request: true

import { HttpMiddlewarePipeline, createRequestContext, createResponseContext, createTimingMiddleware } from "@zudojs/http";

const pipeline = new HttpMiddlewarePipeline({
  middlewares: [createTimingMiddleware(), async () => createResponseContext().json({ status: "ok" })],
});
const response = await pipeline.execute(createRequestContext({ method: "GET", url: "http://localhost/health" }), createResponseContext());
const header = response.headers["server-timing"];
console.log("Server-Timing looks like total;dur=<ms>:", /^total;dur=\d+\.\d{2}$/.test(String(header)));

Server-Timing looks like total;dur=<ms>: true

import { PGlite } from "@electric-sql/pglite";

const db = new PGlite();
await db.exec(`
  CREATE TABLE users (id TEXT PRIMARY KEY, name TEXT NOT NULL);
  CREATE TABLE tasks (id SERIAL PRIMARY KEY, owner_id TEXT NOT NULL, assignee_id TEXT REFERENCES users(id), title TEXT NOT NULL);
  INSERT INTO users SELECT 'u-' || g, 'User ' || g FROM generate_series(1, 200) g;
  INSERT INTO tasks (owner_id, assignee_id, title) SELECT 'u-1', 'u-' || (1 + g % 200), 'Task ' || g FROM generate_series(1, 5000) g;
`);
let queries = 0;
function query<T>(sql: string, params: unknown[] = []) {
  queries += 1;
  return db.query<T>(sql, params);
}

async function boardOneByOne(ownerId: string) {
  const { rows } = await query<{ id: number; title: string; assignee_id: string }>(
    "SELECT id, title, assignee_id FROM tasks WHERE owner_id = $1 ORDER BY id LIMIT 50", [ownerId]);
  return Promise.all(rows.map(async (task) => {
    const user = await query<{ name: string }>("SELECT name FROM users WHERE id = $1", [task.assignee_id]);
    return { id: task.id, title: task.title, assignee: user.rows[0]!.name };
  }));
}

async function boardJoined(ownerId: string) {
  const { rows } = await query<{ id: number; title: string; assignee: string }>(
    `SELECT t.id, t.title, u.name AS assignee FROM tasks t JOIN users u ON u.id = t.assignee_id
     WHERE t.owner_id = $1 ORDER BY t.id LIMIT 50`, [ownerId]);
  return rows;
}

queries = 0;
const slow = await boardOneByOne("u-1");
console.log("one by one:", slow.length, "tasks,", queries, "queries");
queries = 0;
const fast = await boardJoined("u-1");
console.log("joined:", fast.length, "tasks,", queries, "query");
console.log("same result:", JSON.stringify(slow) === JSON.stringify(fast));
await db.close();

one by one: 50 tasks, 51 queries
joined: 50 tasks, 1 query
same result: true

import { PGlite } from "@electric-sql/pglite";

const db = new PGlite();
await db.exec(`
  CREATE TABLE tasks (id SERIAL PRIMARY KEY, owner_id TEXT NOT NULL, title TEXT NOT NULL);
  INSERT INTO tasks (owner_id, title) SELECT 'u-' || (g % 500), 'Task ' || g FROM generate_series(1, 50000) g;
  ANALYZE tasks;
`);
async function plan(label: string, sql: string, params: unknown[] = []) {
  const { rows } = await db.query<{ "QUERY PLAN": string }>(
    `EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF, BUFFERS OFF) ${sql}`, params);
  console.log(label);
  for (const row of rows) console.log("  " + row["QUERY PLAN"]);
}

const page = "SELECT id, title FROM tasks WHERE owner_id = $1 ORDER BY id LIMIT 50";
await plan("1. one user's first page, no index:", page, ["u-42"]);
await db.exec("CREATE INDEX tasks_owner_id_id_idx ON tasks (owner_id, id)");
await plan("2. the same, with an index on (owner_id, id):", page, ["u-42"]);
await plan("3. page 801 of everything with OFFSET:", "SELECT id, title FROM tasks ORDER BY id LIMIT 50 OFFSET 40000");
await plan("4. the same page with a keyset cursor:", "SELECT id, title FROM tasks WHERE id > $1 ORDER BY id LIMIT 50", [40000]);
await db.close();

1. one user's first page, no index:
  Limit (actual rows=50.00 loops=1)
    ->  Index Scan using tasks_pkey on tasks (actual rows=50.00 loops=1)
          Filter: (owner_id = 'u-42'::text)
          Rows Removed by Filter: 24492
          Index Searches: 1
2. the same, with an index on (owner_id, id):
  Limit (actual rows=50.00 loops=1)
    ->  Index Scan using tasks_owner_id_id_idx on tasks (actual rows=50.00 loops=1)
          Index Cond: (owner_id = 'u-42'::text)
          Index Searches: 1
3. page 801 of everything with OFFSET:
  Limit (actual rows=50.00 loops=1)
    ->  Index Scan using tasks_pkey on tasks (actual rows=40050.00 loops=1)
          Index Searches: 1
4. the same page with a keyset cursor:
  Limit (actual rows=50.00 loops=1)
    ->  Index Scan using tasks_pkey on tasks (actual rows=50.00 loops=1)
          Index Cond: (id > 40000)
          Index Searches: 1

function connectionsNeeded(requestsPerSecond: number, queriesPerRequest: number, queryMs: number): number {
  return Math.ceil(requestsPerSecond * queriesPerRequest * (queryMs / 1000));
}

console.log("board, joined query:", connectionsNeeded(400, 1, 5), "connections busy on average");
console.log("board, N+1 (51 queries):", connectionsNeeded(400, 51, 5), "connections busy on average");
console.log("board, 1 slow query of 120 ms:", connectionsNeeded(400, 1, 120), "connections busy on average");

board, joined query: 2 connections busy on average
board, N+1 (51 queries): 102 connections busy on average
board, 1 slow query of 120 ms: 48 connections busy on average

const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));

function createDatabase(cores: number) {
  let active = 0;
  let slowestQuery = 0;
  return {
    get slowestQuery() { return slowestQuery; },
    async query(): Promise<void> {
      active += 1;
      const ms = 20 * Math.max(1, active / cores);
      slowestQuery = Math.max(slowestQuery, ms);
      await sleep(ms);
      active -= 1;
    },
  };
}

function createPool(size: number, db: ReturnType<typeof createDatabase>) {
  let free = size;
  let mostWaiting = 0;
  const waiting: Array<() => void> = [];
  return {
    get mostWaiting() { return mostWaiting; },
    async query(): Promise<void> {
      if (free > 0) free -= 1;
      else await new Promise<void>((resolve) => { waiting.push(resolve); mostWaiting = Math.max(mostWaiting, waiting.length); });
      try {
        await db.query();
      } finally {
        const next = waiting.shift();
        if (next) next();
        else free += 1;
      }
    },
  };
}

async function run(size: number) {
  const db = createDatabase(4);
  const pool = createPool(size, db);
  const start = performance.now();
  await Promise.all(Array.from({ length: 40 }, () => pool.query()));
  return { total: performance.now() - start, mostWaiting: pool.mostWaiting, slowestQuery: db.slowestQuery };
}

const base = await run(4);
for (const size of [2, 4, 8, 40]) {
  const result = size === 4 ? base : await run(size);
  console.log(`pool ${String(size).padStart(2)}: waited in pool ${String(result.mostWaiting).padStart(2)}, slowest query ${String(result.slowestQuery).padStart(3)} ms, total time x${(result.total / base.total).toFixed(0)} of pool 4`);
}

pool  2: waited in pool 38, slowest query  20 ms, total time x2 of pool 4
pool  4: waited in pool 36, slowest query  20 ms, total time x1 of pool 4
pool  8: waited in pool 32, slowest query  40 ms, total time x1 of pool 4
pool 40: waited in pool  0, slowest query 200 ms, total time x1 of pool 4

import { createCacheService, createMemoryCacheAdapter } from "@zudojs/cache";

const cache = createCacheService({ adapter: createMemoryCacheAdapter({ maxEntries: 1_000 }), config: { defaultTtl: 60_000 } });
let databaseLoads = 0;
const load = async (key: string) => {
  databaseLoads += 1;
  return { key };
};

for (let request = 0; request < 1_000; request++) {
  const key = request % 5 < 3 ? "board.launch-week" : request % 5 === 3 ? `tasks.u-${request % 40}` : `task.${request}`;
  await cache.getOrSet(key, () => load(key));
}
console.log(cache.getHotKeys(3));
console.log("hit rate:", cache.getStats()?.hitRate, "database loads:", databaseLoads);

[
  { key: 'zudojs:board.launch-week', hits: 599 },
  { key: 'zudojs:tasks.u-3', hits: 24 },
  { key: 'zudojs:tasks.u-8', hits: 24 }
]
hit rate: 0.791 database loads: 209

import { createCacheService, createMemoryCacheAdapter } from "@zudojs/cache";

let sharedReads = 0;
const shared = new Map<string, unknown>([["board.launch-week", { tasks: 12 }]]);
const readShared = async (key: string) => {
  sharedReads += 1;
  return shared.get(key);
};

const local = createCacheService({ adapter: createMemoryCacheAdapter({ maxEntries: 500 }) });
async function getBoard(): Promise<unknown> {
  const { value } = await local.getOrSet("board.launch-week", () => readShared("board.launch-week"), { ttl: 1_000 });
  return value;
}

await Promise.all(Array.from({ length: 200 }, () => getBoard()));
for (let i = 0; i < 800; i++) await getBoard();
console.log("1,000 requests, reads from the shared cache:", sharedReads);

/** Up to 10% extra, chosen by an FNV-1a hash of the key: the same key always gets the same TTL. */
function jitteredTtl(key: string, baseMs: number): number {
  let hash = 2166136261;
  for (const char of key) hash = Math.imul(hash ^ char.charCodeAt(0), 16777619) >>> 0;
  return baseMs + (hash % 1_000) * (baseMs / 10_000);
}
for (const key of ["board.launch-week", "board.q4-planning", "board.support", "board.hiring"]) {
  console.log(key.padEnd(18), `ttl ${(jitteredTtl(key, 60_000) / 1000).toFixed(1)} s`);
}

1,000 requests, reads from the shared cache: 1
board.launch-week  ttl 62.5 s
board.q4-planning  ttl 60.5 s
board.support      ttl 61.9 s
board.hiring       ttl 64.5 s

import { createInMemoryQueue, createQueueName } from "@zudojs/queue";

const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));

function createMailProvider(connections: number) {
  let busy = 0;
  let queuedAtProvider = 0;
  const waiting: Array<() => void> = [];
  return {
    get queuedAtProvider() { return queuedAtProvider; },
    async send(): Promise<void> {
      if (busy < connections) busy += 1;
      else {
        queuedAtProvider += 1;
        await new Promise<void>((resolve) => waiting.push(resolve));
      }
      await wait(40);
      const next = waiting.shift();
      if (next) next();
      else busy -= 1;
    },
  };
}

async function run(concurrency: number) {
  const provider = createMailProvider(5);
  const queue = createInMemoryQueue<{ taskId: number }>(createQueueName(`reminders-${concurrency}`), { concurrency });
  queue.process("remind", () => provider.send());
  const start = performance.now();
  for (let taskId = 1; taskId <= 40; taskId++) await queue.add("remind", { taskId });
  while ((await queue.getStats()).succeeded < 40) await wait(5);
  const ms = performance.now() - start;
  await queue.close();
  return { ms, queuedAtProvider: provider.queuedAtProvider };
}

let previous = await run(1);
console.log(`concurrency  1: ${previous.queuedAtProvider} jobs waited at the provider`);
for (const [before, concurrency] of [[1, 2], [2, 5], [5, 10], [10, 20]] as const) {
  const result = await run(concurrency);
  const verdict = previous.ms > 1.4 * result.ms ? `faster than ${before}` : `no faster than ${before}`;
  console.log(`concurrency ${String(concurrency).padStart(2)}: ${verdict}, ${result.queuedAtProvider} jobs waited at the provider`);
  previous = result;
}

concurrency  1: 0 jobs waited at the provider
concurrency  2: faster than 1, 0 jobs waited at the provider
concurrency  5: faster than 2, 0 jobs waited at the provider
concurrency 10: no faster than 5, 35 jobs waited at the provider
concurrency 20: no faster than 10, 35 jobs waited at the provider

import { createCacheService, createMemoryCacheAdapter } from "@zudojs/cache";

const unbounded = new Map<string, { kobo: number }>();
const adapter = createMemoryCacheAdapter({ maxEntries: 1_000 });
const bounded = createCacheService({ adapter, config: { defaultTtl: 300_000 } });

for (let request = 1; request <= 20_000; request++) {
  const key = `quote.req-${request}`;
  unbounded.set(key, { kobo: 950_000 });
  await bounded.set(key, { kobo: 950_000 });
}
console.log("unbounded map entries:", unbounded.size);
console.log("bounded cache entries:", await bounded.size());

unbounded map entries: 20000
bounded cache entries: 1000

import { PGlite } from "@electric-sql/pglite";

const db = new PGlite();
await db.exec(`
  CREATE TABLE tasks (id SERIAL PRIMARY KEY, owner_id TEXT NOT NULL, title TEXT NOT NULL);
  CREATE INDEX tasks_owner_id_id_idx ON tasks (owner_id, id);
  INSERT INTO tasks (owner_id, title) SELECT 'u-' || (g % 50), 'Task ' || g FROM generate_series(1, 20000) g;
`);
let queries = 0;
async function listTasks(ownerId: string, after = 0, limit = 50) {
  queries += 1;
  const { rows } = await db.query<{ id: number; title: string }>(
    "SELECT id, title FROM tasks WHERE owner_id = $1 AND id > $2 ORDER BY id LIMIT $3", [ownerId, after, limit]);
  return { items: rows, next: rows.length === limit ? rows.at(-1)!.id : undefined };
}
async function rowsRead(sql: string, params: unknown[]): Promise<number> {
  const { rows } = await db.query<{ "QUERY PLAN": string }>(`EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF, BUFFERS OFF) ${sql}`, params);
  return Math.max(...rows.map((row) => Number(row["QUERY PLAN"].match(/actual rows=([\d.]+)/)?.[1] ?? 0)));
}
function check(name: string, ok: boolean): void {
  console.log(`${ok ? "PASS" : "FAIL"}  ${name}`);
}

queries = 0;
const first = await listTasks("u-7");
const deep = await listTasks("u-7", 19_000);
const body = new TextEncoder().encode(JSON.stringify(first)).length;
check("one query per page", queries === 2);
check("a page is at most 8 KB", body <= 8_192);
check("a deep page reads no more rows than it returns",
  (await rowsRead("SELECT id, title FROM tasks WHERE owner_id = $1 AND id > $2 ORDER BY id LIMIT 50", ["u-7", 19_000])) <= 50);
check("the last page has no next cursor", deep.next === undefined);
await db.close();

PASS  one query per page
PASS  a page is at most 8 KB
PASS  a deep page reads no more rows than it returns
PASS  the last page has no next cursor