Something went wrong. Try again.
Open source agent hosting on the AT Protocol
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179// Load test of the new-user path, the way a room arrives: everyone from this one address, each in their// own browser. Per user: the invite's sign-up page, make the account, sign in through the PDS's OAuth// pages, make an agent on Home, open its chat, send one short message and wait for the reply.//// node scripts/loadtest.ts --code <invite code> --users 20 [--spread 60] [--base https://languid.ai]//// Needs Google Chrome (or set CHROME to a Chromium binary). Every account it makes is a real one on the// service: handles are lt<run><i>, agents lt<run><i>a, listed in the results file for cleanup.
import fs from 'node:fs'import { parseArgs } from 'node:util'import { chromium, type Browser, type Page } from 'playwright-core'
const { values } = parseArgs({ options: { code: { type: 'string' }, users: { type: 'string', default: '5' }, spread: { type: 'string', default: '0' }, base: { type: 'string', default: 'https://languid.ai' }, run: { type: 'string' }, message: { type: 'string', default: 'Say hi in five words.' }, timeout: { type: 'string', default: '120' }, out: { type: 'string' }, headed: { type: 'boolean', default: false }, },})if (!values.code) { console.error('usage: node scripts/loadtest.ts --code <invite code> [--users 5] [--spread seconds] [--base url] [--run tag]') process.exit(2)}const users = Number(values.users)const spreadMs = Number(values.spread) * 1000const stepTimeout = Number(values.timeout) * 1000const base = values.base!.replace(/\/$/, '')// Short, so lt<run><i>a stays within a handle's 20 characters.const run = (values.run ?? Date.now().toString(36).slice(-4)).toLowerCase()const password = `lt-${run}-${Math.random().toString(36).slice(2, 12)}`
const STEPS = ['signupPage', 'signup', 'signIn', 'password', 'authorize', 'createAgent', 'reply'] as consttype Step = (typeof STEPS)[number]
interface Result { i: number handle: string agent: string steps: Partial<Record<Step, number>> failedAt?: Step error?: string /** Responses from the service or the PDS with status >= 400, by step. */ bad: { step: Step; status: number; url: string }[] reply?: string}
class StepError extends Error {}
/** What a page says when it isn't the one expected: its heading and first paragraph. */async function pageSays(p: Page) { return p .evaluate(() => [document.title, document.querySelector('h1,h2')?.textContent, document.querySelector('main p, .error')?.textContent].filter(Boolean).join(' | ')) .catch(() => '')}
async function user(browser: Browser, i: number): Promise<Result> { const handle = `lt${run}${i}` const r: Result = { i, handle, agent: `${handle}a`, steps: {}, bad: [] } const ctx = await browser.newContext() const p = await ctx.newPage() p.setDefaultTimeout(stepTimeout) let step: Step = 'signupPage' p.on('response', (res) => { const u = res.url() if (res.status() >= 400 && (u.startsWith(base) || u.includes('pds.'))) r.bad.push({ step, status: res.status(), url: u.replace(/\?.*/, '') }) }) const time = async (s: Step, fn: () => Promise<unknown>) => { step = s const t = performance.now() try { await fn() } catch (e: any) { r.failedAt = s const says = await pageSays(p) r.error = `${(e?.message ?? String(e)).split('\n')[0].slice(0, 160)}${says ? ` [page: ${says.slice(0, 160)}]` : ''}` throw e } r.steps[s] = performance.now() - t } const expect = async (ok: boolean) => { if (!ok) throw new StepError(`unexpected page ${p.url().replace(/\?.*/, '')}`) } try { await time('signupPage', async () => { const res = await p.goto(`${base}/signup?code=${encodeURIComponent(values.code!)}`) await expect(res?.status() === 200 && (await p.locator('form.signup').count()) === 1) }) await time('signup', async () => { await p.fill('form.signup input[name=handle]', handle) await p.fill('form.signup input[name=password]', password) await Promise.all([p.waitForURL(/\/oauth\/authorize|\/signup/, { waitUntil: 'commit' }).catch(() => {}), p.click('form.signup button')]) await expect(/\/oauth\/authorize/.test(p.url()) || (await p.locator('form.signin').count()) === 1) }) await time('signIn', async () => { if (!/\/oauth\/authorize/.test(p.url())) { await Promise.all([p.waitForURL(/\/oauth\/authorize/, { waitUntil: 'commit' }).catch(() => {}), p.click('form.signin button')]) } await expect(/\/oauth\/authorize/.test(p.url())) await p.locator('input[type=password]').waitFor() }) await time('password', async () => { await p.locator('input[type=password]').fill(password) await p.locator('button[type=submit]').first().click() await p.locator('button[type=submit]', { hasText: 'Authorize' }).waitFor() }) await time('authorize', async () => { await Promise.all([p.waitForURL((u) => u.href.startsWith(base) && !u.pathname.startsWith('/oauth')), p.locator('button[type=submit]', { hasText: 'Authorize' }).click()]) await p.locator('form#new-agent').waitFor() }) await time('createAgent', async () => { await p.fill('form#new-agent input[name=name]', r.agent) await Promise.all([p.waitForNavigation(), p.click('form#new-agent button')]) await expect(p.url().includes(`/agents/${r.agent}`)) await p.locator('#composer').waitFor() }) await time('reply', async () => { await p.fill('#composer [name=text]', values.message!) await p.click('#composer button[type=submit]') const done = p.locator('#log .msg.assistant:not(.live), #log p.error').first() await done.waitFor() if (await done.evaluate((e) => e.matches('p.error'))) throw new StepError(`run failed: ${await done.textContent()}`) r.reply = (await done.textContent())?.trim().slice(0, 80) }) } catch { // Recorded in r. } finally { await ctx.close().catch(() => {}) } return r}
const pct = (xs: number[], q: number) => { if (!xs.length) return NaN const s = [...xs].sort((a, b) => a - b) return s[Math.min(s.length - 1, Math.ceil(q * s.length) - 1)]}const sec = (ms: number) => (Number.isNaN(ms) ? '-' : (ms / 1000).toFixed(1) + 's')
const browser = await chromium.launch(process.env.CHROME ? { executablePath: process.env.CHROME, headless: !values.headed } : { channel: 'chrome', headless: !values.headed })console.log(`run ${run}: ${users} users against ${base}${spreadMs ? ` arriving over ${spreadMs / 1000}s` : ', all at once'}`)const started = performance.now()const results = await Promise.all( Array.from({ length: users }, async (_, i) => { if (spreadMs) await new Promise((ok) => setTimeout(ok, (spreadMs * i) / users)) const r = await user(browser, i) console.log(` ${r.handle.padEnd(12)} ${r.failedAt ? `FAILED at ${r.failedAt}: ${r.error}` : `ok in ${sec(Object.values(r.steps).reduce((a, b) => a + b, 0))}: ${r.reply}`}`) return r }),)await browser.close()const wall = performance.now() - started
console.log(`\n${'step'.padEnd(12)} ${'ok'.padStart(4)} ${'p50'.padStart(7)} ${'p95'.padStart(7)} ${'max'.padStart(7)} ${'fail'.padStart(4)} 4xx/5xx seen`)for (const s of STEPS) { const xs = results.flatMap((r) => (r.steps[s] !== undefined ? [r.steps[s]!] : [])) const fails = results.filter((r) => r.failedAt === s).length const codes = new Map<number, number>() for (const r of results) for (const b of r.bad) if (b.step === s) codes.set(b.status, (codes.get(b.status) ?? 0) + 1) const seen = [...codes].map(([c, n]) => `${c}×${n}`).join(' ') console.log(`${s.padEnd(12)} ${String(xs.length).padStart(4)} ${sec(pct(xs, 0.5)).padStart(7)} ${sec(pct(xs, 0.95)).padStart(7)} ${sec(xs.length ? Math.max(...xs) : NaN).padStart(7)} ${String(fails).padStart(4)} ${seen}`)}const ok = results.filter((r) => !r.failedAt).lengthconst totals = results.filter((r) => !r.failedAt).map((r) => Object.values(r.steps).reduce((a, b) => a + b, 0))console.log(`\n${ok}/${users} made it to a reply; end to end p50 ${sec(pct(totals, 0.5))}, p95 ${sec(pct(totals, 0.95))}; wall ${sec(wall)}`)const errors = new Map<string, number>()for (const r of results) if (r.error) errors.set(`${r.failedAt}: ${r.error.replace(/lt[a-z0-9]+/g, 'lt…')}`, (errors.get(`${r.failedAt}: ${r.error.replace(/lt[a-z0-9]+/g, 'lt…')}`) ?? 0) + 1)for (const [e, n] of errors) console.log(` ${n}× ${e}`)
const out = values.out ?? `loadtest-${run}.json`fs.writeFileSync(out, JSON.stringify({ run, base, users, spreadMs, wallMs: wall, results }, null, 2))console.log(`results (with every handle made): ${out}`)