Files
sim/apps/docs/scripts/test-docs-interfaces-e2e.ts
Waleed 04b66d470d improvement(docs): collect reader feedback inline (#9538)
* improvement(docs): collect reader feedback inline

* fix(docs): validate feedback request authority and exercise retries

* fix(docs): normalize feedback paths and scope error assertions
2026-10-31 12:08:23 -07:00

378 lines
14 KiB
TypeScript

import { mkdir, writeFile } from 'node:fs/promises'
import { dirname } from 'node:path'
import { Client, StreamableHTTPClientTransport } from '@modelcontextprotocol/client'
import { createLogger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
import { generateId } from '@sim/utils/id'
import { toRecord } from '@sim/utils/object'
import routes from '@/.content/routes.json'
const logger = createLogger('DocsInterfacesE2E')
const origin = process.env.DOCS_E2E_BASE_URL
const reportPath = process.env.DOCS_INTERFACES_REPORT_PATH
if (!origin || !reportPath)
throw new Error('DOCS_E2E_BASE_URL and DOCS_INTERFACES_REPORT_PATH are required')
const base = new URL(origin)
if (!['localhost', '127.0.0.1', '[::1]'].includes(base.hostname))
throw new Error('Only a loopback docs server may be tested')
const checks: Array<{
name: string
status: 'passed' | 'failed'
durationMs: number
error?: string
}> = []
function assert(condition: unknown, message: string): asserts condition {
if (!condition) throw new Error(message)
}
async function check(name: string, run: () => Promise<void>) {
const started = performance.now()
try {
await run()
checks.push({ name, status: 'passed', durationMs: Math.round(performance.now() - started) })
} catch (error) {
checks.push({
name,
status: 'failed',
durationMs: Math.round(performance.now() - started),
error: getErrorMessage(error),
})
}
}
async function get(path: string, options?: RequestInit) {
return fetch(new URL(path, base), { ...options, signal: AbortSignal.timeout(20000) })
}
const paths = [
'/introduction',
'/integrations/slack',
'/api-reference/workflows/listWorkflows',
'/cli',
'/mcp/overview',
]
const feedback = {
submissionId: generateId(),
page: '/introduction',
helpful: true,
comment: '',
}
for (const example of [
{
name: 'foreign origin',
origin: 'https://example.com',
contentType: 'application/json',
body: JSON.stringify(feedback),
status: 403,
},
{
name: 'non-JSON content',
origin: base.origin,
contentType: 'text/plain',
body: JSON.stringify(feedback),
status: 415,
},
{
name: 'oversized body',
origin: base.origin,
contentType: 'application/json',
body: JSON.stringify({ ...feedback, comment: 'x'.repeat(13000) }),
status: 413,
},
{
name: 'invalid vote',
origin: base.origin,
contentType: 'application/json',
body: JSON.stringify({ ...feedback, helpful: 'yes' }),
status: 400,
},
{
name: 'unknown page',
origin: base.origin,
contentType: 'application/json',
body: JSON.stringify({ ...feedback, page: '/no-such-document' }),
status: 404,
},
]) {
await check(`Feedback rejects ${example.name} before forwarding`, async () => {
const response = await get('/api/feedback', {
method: 'POST',
headers: { Origin: example.origin, 'Content-Type': example.contentType },
body: example.body,
})
assert(response.status === example.status, `Expected ${example.status}, got ${response.status}`)
assert(
response.headers.get('Cache-Control') === 'no-store',
'Feedback response must not be cached'
)
})
}
for (const path of paths) {
await check(`Complete JSON and Markdown representations agree: ${path}`, async () => {
const json = await get(`/api/docs/page?path=${encodeURIComponent(path)}`)
assert(json.ok, `JSON page returned ${json.status}`)
const page = toRecord(toRecord(await json.json()).page)
assert(
page.path === path && typeof page.markdown === 'string',
'Page identity or content is missing'
)
assert(page.markdown.includes(`(${path})`), 'Markdown lost its canonical identity')
assert(page.markdown.length > 100, 'Page content is incomplete')
for (const suffix of [`${path}.md`, `${path}.mdx`, `/llms.mdx${path}`]) {
const response = await get(suffix)
assert(response.ok, `${suffix} returned ${response.status}`)
assert(
response.headers.get('Content-Type')?.startsWith('text/markdown'),
'Wrong Markdown content type'
)
assert((await response.text()) === page.markdown, 'Representations disagree')
}
if (path.includes('/api-reference/')) {
assert(page.markdown.includes('GET /api/v2/workflows'), 'API operation method/path missing')
assert(page.markdown.includes('Responses'), 'API responses missing')
}
})
}
await check('Discovery pagination has no gaps, duplicates or unbounded pages', async () => {
const seen = new Set<string>()
const paths = new Set<string>()
let cursor: string | null = null
let total = 0
for (let request = 0; request < 100; request += 1) {
const response = await get(
`/api/docs/pages?limit=100${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`
)
assert(response.ok, `Discovery returned ${response.status}`)
const data = toRecord(await response.json())
assert(Array.isArray(data.pages) && data.pages.length <= 100, 'Unbounded discovery response')
for (const value of data.pages) {
const page = toRecord(value)
assert(typeof page.id === 'string' && !seen.has(page.id), 'Duplicate or missing identity')
seen.add(page.id)
assert(typeof page.path === 'string', 'Discovery path is missing')
paths.add(page.path)
}
assert(typeof data.total === 'number', 'Missing discovery total')
total = data.total
cursor = typeof data.nextCursor === 'string' ? data.nextCursor : null
if (!cursor) break
}
const canonicalPaths = routes.filter((path) => path !== '/')
assert(
cursor === null &&
seen.size === total &&
paths.size === canonicalPaths.length &&
canonicalPaths.every((path) => paths.has(path)),
'Discovery does not match the complete canonical route catalog'
)
for (const query of ['limit=101', 'limit=-1', 'section=unknown', 'cursor=unknown']) {
assert((await get(`/api/docs/pages?${query}`)).status === 400, `Invalid ${query} accepted`)
}
})
await check('Canonical page lookup normalizes a trailing slash', async () => {
const response = await get('/api/docs/page?path=%2Fintroduction%2F')
assert(response.ok, 'Trailing-slash page lookup failed')
const page = toRecord(toRecord(await response.json()).page)
assert(page.path === '/introduction', 'Page identity was not canonicalized')
})
await check('Malformed canonical paths are rejected without blocking the HTTP worker', async () => {
const malformed = `/${'a'.repeat(32)}!`
const response = await fetch(
new URL(`/api/docs/page?path=${encodeURIComponent(malformed)}`, base),
{
signal: AbortSignal.timeout(2000),
}
)
assert(response.status === 400, 'Malformed path was not rejected')
})
await check('Unknown pages and arbitrary URL inputs fail explicitly', async () => {
assert(
(await get('/api/docs/page?path=/no-such-document')).status === 404,
'Unknown JSON page did not return404'
)
assert(
(await get('/no-such-document.mdx')).status === 404,
'Unknown Markdown page did not return404'
)
for (const path of ['https://example.com', '//example.com', '/../secret', '/%2e%2e/secret']) {
assert(
(await get(`/api/docs/page?path=${encodeURIComponent(path)}`)).status === 400,
`Non-canonical path accepted: ${path}`
)
}
})
for (const entry of [
{ query: 'Slack', prefix: '/integrations/' },
{ query: 'slack send message', prefix: '/integrations/slack' },
{ query: 'List workflows', prefix: '/api-reference/' },
{ query: 'workflow variables', prefix: '/workflows/variables' },
{ query: 'webhook trigger', prefix: '/workflows/blocks/webhook' },
{ query: 'CLI', prefix: '/cli' },
{ query: 'MCP', prefix: '/mcp/' },
]) {
await check(`Credential-free search covers ${entry.prefix}`, async () => {
const response = await get(`/api/search?q=${encodeURIComponent(entry.query)}&limit=20`)
assert(response.ok, `Search returned ${response.status}`)
const result: unknown = await response.json()
assert(
Array.isArray(result) && result.length > 0 && result.length <= 20,
'Search is empty or unbounded'
)
assert(
result.some((value) => {
const url = toRecord(value).url
return typeof url === 'string' && new URL(url).pathname.startsWith(entry.prefix)
}),
'Expected section missing from search'
)
for (const value of result) {
const row = toRecord(value)
assert(
row.type === 'page' &&
typeof row.id === 'string' &&
typeof row.content === 'string' &&
Array.isArray(row.breadcrumbs),
'Search wire contract changed'
)
}
})
}
await check('Search keeps empty, bounded, and alias behavior', async () => {
const empty = await get('/api/search?query=')
assert(JSON.stringify(await empty.json()) === '[]', 'Empty search is not an empty list')
const query = await get('/api/search?query=slack&limit=1')
const alias = await get('/api/search?q=slack&limit=1')
const first: unknown = await query.json()
assert(Array.isArray(first) && first.length === 1, 'Search limit ignored')
assert(JSON.stringify(first) === JSON.stringify(await alias.json()), 'query and q disagree')
for (let attempt = 0; attempt < 12; attempt += 1) {
const term = attempt % 2 ? `zzzz-no-such-content-${attempt}-zzzz` : 'Slack'
const response = await fetch(new URL(`/api/search?query=${term}&attempt=${attempt}`, base), {
signal: AbortSignal.timeout(3000),
cache: 'no-store',
})
assert(response.ok, `Repeated search ${attempt} returned ${response.status}`)
const results: unknown = await response.json()
assert(Array.isArray(results), 'Repeated search did not return results')
assert(
attempt % 2
? results.length === 0
: results.some((value) => {
const url = toRecord(value).url
return typeof url === 'string' && new URL(url).pathname === '/integrations/slack'
}),
`Repeated search ${attempt} lost its catalog fallback`
)
}
})
await check(
'MCP rejects untrusted origins and oversized bodies before executing tools',
async () => {
const payload = JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/list' })
const denied = await get('/api/mcp', {
method: 'POST',
headers: {
Origin: 'https://untrusted.example',
'Content-Type': 'application/json',
Accept: 'application/json, text/event-stream',
},
body: payload,
})
assert(denied.status === 403, `Untrusted origin returned ${denied.status}`)
const large = await get('/api/mcp', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Accept: 'application/json, text/event-stream',
},
body: ' '.repeat(20 * 1024),
})
assert(large.status === 413, `Oversized body returned ${large.status}`)
}
)
const client = new Client({ name: 'sim-docs-e2e', version: '1.0.0' })
try {
await check(
'Official MCP client connects, discovers and invokes all read-only tools',
async () => {
await client.connect(new StreamableHTTPClientTransport(new URL('/api/mcp', base)))
const tools = await client.listTools()
for (const name of ['list_pages', 'get_page', 'search']) {
assert(
tools.tools.some((tool) => tool.name === name && tool.annotations?.readOnlyHint),
`${name} was not advertised as read-only`
)
}
const discovery = await client.callTool({ name: 'list_pages', arguments: { limit: 2 } })
const discovered = toRecord(discovery.structuredContent)
assert(
Array.isArray(discovered.pages) && discovered.pages.length === 2 && discovered.nextCursor,
'MCP discovery pagination failed'
)
const page = await client.callTool({
name: 'get_page',
arguments: { path: '/integrations/slack' },
})
const fetched = toRecord(toRecord(page.structuredContent).page)
const canonical = toRecord(
toRecord(await (await get('/api/docs/page?path=/integrations/slack')).json()).page
)
assert(
fetched.markdown === canonical.markdown && fetched.url === canonical.url,
'MCP page was truncated or changed'
)
const result = await client.callTool({
name: 'search',
arguments: { query: 'Slack', limit: 3 },
})
const http = await (await get('/api/search?query=Slack&limit=3')).json()
assert(
JSON.stringify(toRecord(result.structuredContent).results) === JSON.stringify(http),
'MCP and HTTP search disagree'
)
}
)
await check(
'MCP unknown pages and invalid inputs return errors through the real client',
async () => {
const missing = await client.callTool({
name: 'get_page',
arguments: { path: '/no-such-document' },
})
assert(missing.isError, 'Unknown MCP page reported success')
for (const input of [
{ name: 'get_page', arguments: { path: 'https://example.com' } },
{ name: 'list_pages', arguments: { limit: 101 } },
]) {
let rejected = false
try {
rejected = Boolean((await client.callTool(input)).isError)
} catch (error) {
if (toRecord(error).code !== -32602) throw error
rejected = true
}
assert(rejected, 'Invalid MCP input reported success')
}
}
)
} finally {
await client.close()
await mkdir(dirname(reportPath), { recursive: true })
await writeFile(reportPath, `${JSON.stringify({ checks }, null, 2)}\n`)
}
const failed = checks.filter((entry) => entry.status === 'failed')
if (failed.length) {
logger.error('Documentation interfaces failed', { failed })
process.exitCode = 1
} else logger.info('Documentation interfaces passed', { checks: checks.length })