mirror of
https://github.com/simstudioai/sim.git
synced 2026-11-20 12:08:01 +08:00
* improvement(docs): collect reader feedback inline * fix(docs): validate feedback request authority and exercise retries * fix(docs): normalize feedback paths and scope error assertions
378 lines
14 KiB
TypeScript
378 lines
14 KiB
TypeScript
import { mkdir, writeFile } from 'node:fs/promises'
|
|
import { dirname } from 'node:path'
|
|
import { Client, StreamableHTTPClientTransport } from '@modelcontextprotocol/client'
|
|
import { createLogger } from '@sim/logger'
|
|
import { getErrorMessage } from '@sim/utils/errors'
|
|
import { generateId } from '@sim/utils/id'
|
|
import { toRecord } from '@sim/utils/object'
|
|
import routes from '@/.content/routes.json'
|
|
|
|
const logger = createLogger('DocsInterfacesE2E')
|
|
const origin = process.env.DOCS_E2E_BASE_URL
|
|
const reportPath = process.env.DOCS_INTERFACES_REPORT_PATH
|
|
if (!origin || !reportPath)
|
|
throw new Error('DOCS_E2E_BASE_URL and DOCS_INTERFACES_REPORT_PATH are required')
|
|
const base = new URL(origin)
|
|
if (!['localhost', '127.0.0.1', '[::1]'].includes(base.hostname))
|
|
throw new Error('Only a loopback docs server may be tested')
|
|
const checks: Array<{
|
|
name: string
|
|
status: 'passed' | 'failed'
|
|
durationMs: number
|
|
error?: string
|
|
}> = []
|
|
|
|
function assert(condition: unknown, message: string): asserts condition {
|
|
if (!condition) throw new Error(message)
|
|
}
|
|
|
|
async function check(name: string, run: () => Promise<void>) {
|
|
const started = performance.now()
|
|
try {
|
|
await run()
|
|
checks.push({ name, status: 'passed', durationMs: Math.round(performance.now() - started) })
|
|
} catch (error) {
|
|
checks.push({
|
|
name,
|
|
status: 'failed',
|
|
durationMs: Math.round(performance.now() - started),
|
|
error: getErrorMessage(error),
|
|
})
|
|
}
|
|
}
|
|
|
|
async function get(path: string, options?: RequestInit) {
|
|
return fetch(new URL(path, base), { ...options, signal: AbortSignal.timeout(20000) })
|
|
}
|
|
|
|
const paths = [
|
|
'/introduction',
|
|
'/integrations/slack',
|
|
'/api-reference/workflows/listWorkflows',
|
|
'/cli',
|
|
'/mcp/overview',
|
|
]
|
|
const feedback = {
|
|
submissionId: generateId(),
|
|
page: '/introduction',
|
|
helpful: true,
|
|
comment: '',
|
|
}
|
|
for (const example of [
|
|
{
|
|
name: 'foreign origin',
|
|
origin: 'https://example.com',
|
|
contentType: 'application/json',
|
|
body: JSON.stringify(feedback),
|
|
status: 403,
|
|
},
|
|
{
|
|
name: 'non-JSON content',
|
|
origin: base.origin,
|
|
contentType: 'text/plain',
|
|
body: JSON.stringify(feedback),
|
|
status: 415,
|
|
},
|
|
{
|
|
name: 'oversized body',
|
|
origin: base.origin,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({ ...feedback, comment: 'x'.repeat(13000) }),
|
|
status: 413,
|
|
},
|
|
{
|
|
name: 'invalid vote',
|
|
origin: base.origin,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({ ...feedback, helpful: 'yes' }),
|
|
status: 400,
|
|
},
|
|
{
|
|
name: 'unknown page',
|
|
origin: base.origin,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({ ...feedback, page: '/no-such-document' }),
|
|
status: 404,
|
|
},
|
|
]) {
|
|
await check(`Feedback rejects ${example.name} before forwarding`, async () => {
|
|
const response = await get('/api/feedback', {
|
|
method: 'POST',
|
|
headers: { Origin: example.origin, 'Content-Type': example.contentType },
|
|
body: example.body,
|
|
})
|
|
assert(response.status === example.status, `Expected ${example.status}, got ${response.status}`)
|
|
assert(
|
|
response.headers.get('Cache-Control') === 'no-store',
|
|
'Feedback response must not be cached'
|
|
)
|
|
})
|
|
}
|
|
|
|
for (const path of paths) {
|
|
await check(`Complete JSON and Markdown representations agree: ${path}`, async () => {
|
|
const json = await get(`/api/docs/page?path=${encodeURIComponent(path)}`)
|
|
assert(json.ok, `JSON page returned ${json.status}`)
|
|
const page = toRecord(toRecord(await json.json()).page)
|
|
assert(
|
|
page.path === path && typeof page.markdown === 'string',
|
|
'Page identity or content is missing'
|
|
)
|
|
assert(page.markdown.includes(`(${path})`), 'Markdown lost its canonical identity')
|
|
assert(page.markdown.length > 100, 'Page content is incomplete')
|
|
for (const suffix of [`${path}.md`, `${path}.mdx`, `/llms.mdx${path}`]) {
|
|
const response = await get(suffix)
|
|
assert(response.ok, `${suffix} returned ${response.status}`)
|
|
assert(
|
|
response.headers.get('Content-Type')?.startsWith('text/markdown'),
|
|
'Wrong Markdown content type'
|
|
)
|
|
assert((await response.text()) === page.markdown, 'Representations disagree')
|
|
}
|
|
if (path.includes('/api-reference/')) {
|
|
assert(page.markdown.includes('GET /api/v2/workflows'), 'API operation method/path missing')
|
|
assert(page.markdown.includes('Responses'), 'API responses missing')
|
|
}
|
|
})
|
|
}
|
|
|
|
await check('Discovery pagination has no gaps, duplicates or unbounded pages', async () => {
|
|
const seen = new Set<string>()
|
|
const paths = new Set<string>()
|
|
let cursor: string | null = null
|
|
let total = 0
|
|
for (let request = 0; request < 100; request += 1) {
|
|
const response = await get(
|
|
`/api/docs/pages?limit=100${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`
|
|
)
|
|
assert(response.ok, `Discovery returned ${response.status}`)
|
|
const data = toRecord(await response.json())
|
|
assert(Array.isArray(data.pages) && data.pages.length <= 100, 'Unbounded discovery response')
|
|
for (const value of data.pages) {
|
|
const page = toRecord(value)
|
|
assert(typeof page.id === 'string' && !seen.has(page.id), 'Duplicate or missing identity')
|
|
seen.add(page.id)
|
|
assert(typeof page.path === 'string', 'Discovery path is missing')
|
|
paths.add(page.path)
|
|
}
|
|
assert(typeof data.total === 'number', 'Missing discovery total')
|
|
total = data.total
|
|
cursor = typeof data.nextCursor === 'string' ? data.nextCursor : null
|
|
if (!cursor) break
|
|
}
|
|
const canonicalPaths = routes.filter((path) => path !== '/')
|
|
assert(
|
|
cursor === null &&
|
|
seen.size === total &&
|
|
paths.size === canonicalPaths.length &&
|
|
canonicalPaths.every((path) => paths.has(path)),
|
|
'Discovery does not match the complete canonical route catalog'
|
|
)
|
|
for (const query of ['limit=101', 'limit=-1', 'section=unknown', 'cursor=unknown']) {
|
|
assert((await get(`/api/docs/pages?${query}`)).status === 400, `Invalid ${query} accepted`)
|
|
}
|
|
})
|
|
|
|
await check('Canonical page lookup normalizes a trailing slash', async () => {
|
|
const response = await get('/api/docs/page?path=%2Fintroduction%2F')
|
|
assert(response.ok, 'Trailing-slash page lookup failed')
|
|
const page = toRecord(toRecord(await response.json()).page)
|
|
assert(page.path === '/introduction', 'Page identity was not canonicalized')
|
|
})
|
|
|
|
await check('Malformed canonical paths are rejected without blocking the HTTP worker', async () => {
|
|
const malformed = `/${'a'.repeat(32)}!`
|
|
const response = await fetch(
|
|
new URL(`/api/docs/page?path=${encodeURIComponent(malformed)}`, base),
|
|
{
|
|
signal: AbortSignal.timeout(2000),
|
|
}
|
|
)
|
|
assert(response.status === 400, 'Malformed path was not rejected')
|
|
})
|
|
|
|
await check('Unknown pages and arbitrary URL inputs fail explicitly', async () => {
|
|
assert(
|
|
(await get('/api/docs/page?path=/no-such-document')).status === 404,
|
|
'Unknown JSON page did not return404'
|
|
)
|
|
assert(
|
|
(await get('/no-such-document.mdx')).status === 404,
|
|
'Unknown Markdown page did not return404'
|
|
)
|
|
for (const path of ['https://example.com', '//example.com', '/../secret', '/%2e%2e/secret']) {
|
|
assert(
|
|
(await get(`/api/docs/page?path=${encodeURIComponent(path)}`)).status === 400,
|
|
`Non-canonical path accepted: ${path}`
|
|
)
|
|
}
|
|
})
|
|
|
|
for (const entry of [
|
|
{ query: 'Slack', prefix: '/integrations/' },
|
|
{ query: 'slack send message', prefix: '/integrations/slack' },
|
|
{ query: 'List workflows', prefix: '/api-reference/' },
|
|
{ query: 'workflow variables', prefix: '/workflows/variables' },
|
|
{ query: 'webhook trigger', prefix: '/workflows/blocks/webhook' },
|
|
{ query: 'CLI', prefix: '/cli' },
|
|
{ query: 'MCP', prefix: '/mcp/' },
|
|
]) {
|
|
await check(`Credential-free search covers ${entry.prefix}`, async () => {
|
|
const response = await get(`/api/search?q=${encodeURIComponent(entry.query)}&limit=20`)
|
|
assert(response.ok, `Search returned ${response.status}`)
|
|
const result: unknown = await response.json()
|
|
assert(
|
|
Array.isArray(result) && result.length > 0 && result.length <= 20,
|
|
'Search is empty or unbounded'
|
|
)
|
|
assert(
|
|
result.some((value) => {
|
|
const url = toRecord(value).url
|
|
return typeof url === 'string' && new URL(url).pathname.startsWith(entry.prefix)
|
|
}),
|
|
'Expected section missing from search'
|
|
)
|
|
for (const value of result) {
|
|
const row = toRecord(value)
|
|
assert(
|
|
row.type === 'page' &&
|
|
typeof row.id === 'string' &&
|
|
typeof row.content === 'string' &&
|
|
Array.isArray(row.breadcrumbs),
|
|
'Search wire contract changed'
|
|
)
|
|
}
|
|
})
|
|
}
|
|
|
|
await check('Search keeps empty, bounded, and alias behavior', async () => {
|
|
const empty = await get('/api/search?query=')
|
|
assert(JSON.stringify(await empty.json()) === '[]', 'Empty search is not an empty list')
|
|
const query = await get('/api/search?query=slack&limit=1')
|
|
const alias = await get('/api/search?q=slack&limit=1')
|
|
const first: unknown = await query.json()
|
|
assert(Array.isArray(first) && first.length === 1, 'Search limit ignored')
|
|
assert(JSON.stringify(first) === JSON.stringify(await alias.json()), 'query and q disagree')
|
|
for (let attempt = 0; attempt < 12; attempt += 1) {
|
|
const term = attempt % 2 ? `zzzz-no-such-content-${attempt}-zzzz` : 'Slack'
|
|
const response = await fetch(new URL(`/api/search?query=${term}&attempt=${attempt}`, base), {
|
|
signal: AbortSignal.timeout(3000),
|
|
cache: 'no-store',
|
|
})
|
|
assert(response.ok, `Repeated search ${attempt} returned ${response.status}`)
|
|
const results: unknown = await response.json()
|
|
assert(Array.isArray(results), 'Repeated search did not return results')
|
|
assert(
|
|
attempt % 2
|
|
? results.length === 0
|
|
: results.some((value) => {
|
|
const url = toRecord(value).url
|
|
return typeof url === 'string' && new URL(url).pathname === '/integrations/slack'
|
|
}),
|
|
`Repeated search ${attempt} lost its catalog fallback`
|
|
)
|
|
}
|
|
})
|
|
|
|
await check(
|
|
'MCP rejects untrusted origins and oversized bodies before executing tools',
|
|
async () => {
|
|
const payload = JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/list' })
|
|
const denied = await get('/api/mcp', {
|
|
method: 'POST',
|
|
headers: {
|
|
Origin: 'https://untrusted.example',
|
|
'Content-Type': 'application/json',
|
|
Accept: 'application/json, text/event-stream',
|
|
},
|
|
body: payload,
|
|
})
|
|
assert(denied.status === 403, `Untrusted origin returned ${denied.status}`)
|
|
const large = await get('/api/mcp', {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json',
|
|
Accept: 'application/json, text/event-stream',
|
|
},
|
|
body: ' '.repeat(20 * 1024),
|
|
})
|
|
assert(large.status === 413, `Oversized body returned ${large.status}`)
|
|
}
|
|
)
|
|
|
|
const client = new Client({ name: 'sim-docs-e2e', version: '1.0.0' })
|
|
try {
|
|
await check(
|
|
'Official MCP client connects, discovers and invokes all read-only tools',
|
|
async () => {
|
|
await client.connect(new StreamableHTTPClientTransport(new URL('/api/mcp', base)))
|
|
const tools = await client.listTools()
|
|
for (const name of ['list_pages', 'get_page', 'search']) {
|
|
assert(
|
|
tools.tools.some((tool) => tool.name === name && tool.annotations?.readOnlyHint),
|
|
`${name} was not advertised as read-only`
|
|
)
|
|
}
|
|
const discovery = await client.callTool({ name: 'list_pages', arguments: { limit: 2 } })
|
|
const discovered = toRecord(discovery.structuredContent)
|
|
assert(
|
|
Array.isArray(discovered.pages) && discovered.pages.length === 2 && discovered.nextCursor,
|
|
'MCP discovery pagination failed'
|
|
)
|
|
const page = await client.callTool({
|
|
name: 'get_page',
|
|
arguments: { path: '/integrations/slack' },
|
|
})
|
|
const fetched = toRecord(toRecord(page.structuredContent).page)
|
|
const canonical = toRecord(
|
|
toRecord(await (await get('/api/docs/page?path=/integrations/slack')).json()).page
|
|
)
|
|
assert(
|
|
fetched.markdown === canonical.markdown && fetched.url === canonical.url,
|
|
'MCP page was truncated or changed'
|
|
)
|
|
const result = await client.callTool({
|
|
name: 'search',
|
|
arguments: { query: 'Slack', limit: 3 },
|
|
})
|
|
const http = await (await get('/api/search?query=Slack&limit=3')).json()
|
|
assert(
|
|
JSON.stringify(toRecord(result.structuredContent).results) === JSON.stringify(http),
|
|
'MCP and HTTP search disagree'
|
|
)
|
|
}
|
|
)
|
|
await check(
|
|
'MCP unknown pages and invalid inputs return errors through the real client',
|
|
async () => {
|
|
const missing = await client.callTool({
|
|
name: 'get_page',
|
|
arguments: { path: '/no-such-document' },
|
|
})
|
|
assert(missing.isError, 'Unknown MCP page reported success')
|
|
for (const input of [
|
|
{ name: 'get_page', arguments: { path: 'https://example.com' } },
|
|
{ name: 'list_pages', arguments: { limit: 101 } },
|
|
]) {
|
|
let rejected = false
|
|
try {
|
|
rejected = Boolean((await client.callTool(input)).isError)
|
|
} catch (error) {
|
|
if (toRecord(error).code !== -32602) throw error
|
|
rejected = true
|
|
}
|
|
assert(rejected, 'Invalid MCP input reported success')
|
|
}
|
|
}
|
|
)
|
|
} finally {
|
|
await client.close()
|
|
await mkdir(dirname(reportPath), { recursive: true })
|
|
await writeFile(reportPath, `${JSON.stringify({ checks }, null, 2)}\n`)
|
|
}
|
|
const failed = checks.filter((entry) => entry.status === 'failed')
|
|
if (failed.length) {
|
|
logger.error('Documentation interfaces failed', { failed })
|
|
process.exitCode = 1
|
|
} else logger.info('Documentation interfaces passed', { checks: checks.length })
|