hermes-agent/apps/desktop/scripts/measure-submit.mjs

241 lines
8.4 KiB
JavaScript

#!/usr/bin/env node
// Measure submit (Enter) latency in the composer.
//
// For each round:
// 1. Focus composer, type N chars of stub text
// 2. Mark a timestamp, fire Enter via Input.dispatchKeyEvent
// 3. Observe: time until the composer becomes empty (submit accepted),
// time until the user message renders in the thread viewport,
// time until the optional "running…" indicator appears,
// time until the next frame is painted after the message renders.
//
// Pre-condition: a session is loaded (load via click-session.mjs first).
// Note: this DOES talk to the real gateway/agent, so each round triggers
// a real prompt submission. Don't run this on a live conversation
// you care about — use a throwaway session.
import { writeFileSync } from 'node:fs'
const args = Object.fromEntries(
process.argv.slice(2).flatMap(s => {
const m = s.match(/^--([^=]+)(?:=(.*))?$/)
return m ? [[m[1], m[2] ?? true]] : []
})
)
const PORT = Number(args.port ?? 9222)
const ROUNDS = Number(args.rounds ?? 3)
async function pickRenderer() {
const list = await (await fetch(`http://127.0.0.1:${PORT}/json/list`)).json()
return list.find(t => t.type === 'page' && t.url.startsWith('http'))
}
function connect(url) {
return new Promise((resolve, reject) => {
const ws = new WebSocket(url)
let id = 0
const pending = new Map()
ws.addEventListener('open', () =>
resolve({
send(method, params = {}) {
const myId = ++id
ws.send(JSON.stringify({ id: myId, method, params }))
return new Promise((res, rej) => pending.set(myId, { res, rej }))
},
close: () => ws.close()
})
)
ws.addEventListener('error', reject)
ws.addEventListener('message', ev => {
const m = JSON.parse(typeof ev.data === 'string' ? ev.data : ev.data.toString('utf8'))
if (m.id != null) {
const p = pending.get(m.id)
if (!p) return
pending.delete(m.id)
m.error ? p.rej(new Error(m.error.message)) : p.res(m.result)
}
})
})
}
async function evalP(cdp, expr) {
const r = await cdp.send('Runtime.evaluate', { expression: expr, returnByValue: true, awaitPromise: true })
if (r.exceptionDetails) throw new Error(r.exceptionDetails.text)
return r.result.value
}
async function focusAndType(cdp, text) {
await evalP(cdp, `
(() => {
const el = document.querySelector('[data-slot="composer-rich-input"]')
if (!el) return
el.focus()
const range = document.createRange()
range.selectNodeContents(el)
range.collapse(false)
const sel = window.getSelection()
sel.removeAllRanges()
sel.addRange(range)
})()
`)
for (const c of text) {
await cdp.send('Input.dispatchKeyEvent', { type: 'char', text: c, unmodifiedText: c })
await new Promise(r => setTimeout(r, 8))
}
}
async function submitAndMeasure(cdp, timeoutMs = 5000) {
// Install observers, await milestones. Returns a promise that we'll resolve
// after the page-side Promise resolves OR a CDP-side timeout (belt + braces).
const setup = await evalP(
cdp,
`
(() => {
const composer = document.querySelector('[data-slot="composer-rich-input"]')
const threadRoot = document.querySelector('[data-slot="aui_thread-content"]') ||
document.querySelector('[data-slot="aui_thread-viewport"]')
const startMessageCount = threadRoot ? threadRoot.querySelectorAll('[data-slot="aui_turn-pair"], [data-slot="aui_message"]').length : 0
window.__submitMilestones = { start: performance.now(), startMessageCount }
window.__submitDone = false
window.__submitResolve = null
const composerObs = new MutationObserver(() => {
const m = window.__submitMilestones
if (!m) return
if (!m.composerClearedMs && composer && composer.innerText.length === 0) {
m.composerClearedMs = performance.now() - m.start
}
})
composer && composerObs.observe(composer, { childList: true, subtree: true, characterData: true })
const threadObs = threadRoot ? new MutationObserver(() => {
const m = window.__submitMilestones
if (!m || m.userMessageRenderedMs) return
const c = threadRoot.querySelectorAll('[data-slot="aui_turn-pair"], [data-slot="aui_message"]').length
if (c > m.startMessageCount) {
m.userMessageRenderedMs = performance.now() - m.start
requestAnimationFrame(() => {
m.userMessagePaintMs = performance.now() - m.start
})
}
}) : null
threadObs && threadObs.observe(threadRoot, { childList: true, subtree: true })
window.__submitObservers = { composerObs, threadObs }
return true
})()
`
)
if (!setup) throw new Error('observer setup failed')
// Send Enter via real keystroke channel (rawKeyDown + char + keyUp).
// React synthetic onKeyDown receives this exactly like a hardware Enter.
await cdp.send('Input.dispatchKeyEvent', {
type: 'rawKeyDown',
windowsVirtualKeyCode: 13,
nativeVirtualKeyCode: 13,
key: 'Enter',
code: 'Enter',
text: '\r',
unmodifiedText: '\r'
})
await cdp.send('Input.dispatchKeyEvent', {
type: 'keyUp',
windowsVirtualKeyCode: 13,
nativeVirtualKeyCode: 13,
key: 'Enter',
code: 'Enter'
})
// Poll for the milestones from outside; cap at timeoutMs.
const deadline = Date.now() + timeoutMs
while (Date.now() < deadline) {
const m = await evalP(cdp, 'JSON.stringify(window.__submitMilestones)')
const parsed = JSON.parse(m || '{}')
if (parsed.userMessagePaintMs != null) {
parsed.reason = 'paint'
await evalP(cdp, `(() => {
window.__submitObservers?.composerObs.disconnect()
window.__submitObservers?.threadObs?.disconnect()
})()`)
return parsed
}
await new Promise(r => setTimeout(r, 50))
}
// Timed out
const m = await evalP(cdp, 'JSON.stringify(window.__submitMilestones)')
const parsed = JSON.parse(m || '{}')
parsed.reason = 'timeout-overall'
await evalP(cdp, `(() => {
window.__submitObservers?.composerObs.disconnect()
window.__submitObservers?.threadObs?.disconnect()
})()`)
return parsed
}
async function tryCancel(cdp) {
// Find a Stop / Cancel button and click it. After submit, the composer
// turns into "cancel" mode; clicking it interrupts the agent turn so we
// don't burn tokens on these probes.
await evalP(
cdp,
`
(() => {
// Common selectors: aria-label="Stop response", data-slot="composer-cancel",
// role=button with text "Stop"
const candidates = [
'[aria-label="Stop response"]',
'[aria-label*="Stop"]',
'[aria-label*="Cancel"]',
'[data-slot*="cancel"]',
'[data-slot*="stop"]'
]
for (const sel of candidates) {
const el = document.querySelector(sel)
if (el) { el.click(); return { clicked: sel } }
}
// Fallback: any button whose textContent includes "Stop"
for (const b of document.querySelectorAll('button')) {
if ((b.textContent || '').toLowerCase().includes('stop')) { b.click(); return { clicked: 'btn-text-Stop' } }
}
return { clicked: null }
})()
`
)
}
async function main() {
const tgt = await pickRenderer()
console.log('target', tgt.url)
const cdp = await connect(tgt.webSocketDebuggerUrl)
await cdp.send('Runtime.enable')
const samples = []
for (let i = 1; i <= ROUNDS; i++) {
await focusAndType(cdp, `latency test ${i} ${'x'.repeat(40)}`)
await new Promise(r => setTimeout(r, 300))
const result = await submitAndMeasure(cdp, 4000)
samples.push({ round: i, ...result })
console.log(
`r${i}: clear=${(result.composerClearedMs ?? -1).toFixed?.(0) ?? '?'}ms ` +
`userMsg=${(result.userMessageRenderedMs ?? -1).toFixed?.(0) ?? '?'}ms ` +
`paint=${(result.userMessagePaintMs ?? -1).toFixed?.(0) ?? '?'}ms ` +
`reason=${result.reason}`
)
// Interrupt the running agent turn so we don't burn tokens on these probes.
await new Promise(r => setTimeout(r, 200))
await tryCancel(cdp)
// Wait long enough for the cancel to settle before next round.
await new Promise(r => setTimeout(r, 1500))
}
writeFileSync('/tmp/hermes-submit-latency.json', JSON.stringify(samples, null, 2))
console.log('\nwrote /tmp/hermes-submit-latency.json')
cdp.close()
}
main().catch(e => {
console.error('fatal:', e.stack ?? e.message)
process.exit(1)
})