Files
sim/apps/docs/scripts/test-search-postgres-e2e.ts
Waleed ced5556efd fix(docs): keep search responsive and contain sidebar scrolling (#9786)
* fix(docs): keep search responsive and contain sidebar scrolling

* fix(docs): strengthen search cancellation and IPv6 checks
2026-11-07 13:58:05 -08:00

833 lines
33 KiB
TypeScript

import assert from 'node:assert/strict'
import { spawn } from 'node:child_process'
import { createHash } from 'node:crypto'
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'
import { connect, createServer, type Socket } from 'node:net'
import { dirname, join, resolve } from 'node:path'
import { createLogger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
import { sleep } from '@sim/utils/helpers'
import { generateId } from '@sim/utils/id'
import { type Server, serve } from 'bun'
import { toString as markdownToString } from 'mdast-util-to-string'
import postgres from 'postgres'
import remarkGfm from 'remark-gfm'
import remarkParse from 'remark-parse'
import { unified } from 'unified'
import { visit } from 'unist-util-visit'
import { z } from 'zod'
import { getPage, getPages } from '@/lib/content/catalog'
import { pageSearchChunks } from '@/lib/search/chunks'
import { searchDatabase } from '@/lib/search/database'
import { searchDocs } from '@/lib/search/service'
const logger = createLogger('DocsSearchPostgresE2E')
const chunksOnly = process.argv.includes('--chunks-only')
const databaseSetting =
process.env.DOCS_SEARCH_E2E_DATABASE_URL ??
(chunksOnly ? 'postgres://127.0.0.1:1/docs_search_test' : undefined)
const reportSetting = process.env.DOCS_SEARCH_E2E_REPORT_PATH
assert(
databaseSetting && reportSetting,
'Explicit disposable database and report paths are required'
)
const databaseUrl = new URL(databaseSetting)
assert(
['localhost', '127.0.0.1', '[::1]'].includes(databaseUrl.hostname),
'Only loopback databases are permitted'
)
assert(databaseUrl.pathname === '/docs_search_test', 'Use a dedicated docs_search_test database')
assert(['postgres:', 'postgresql:'].includes(databaseUrl.protocol), 'A PostgreSQL URL is required')
process.env.DATABASE_URL = databaseUrl.href
process.env.OPENAI_API_KEY = ''
const reportPath: string = reportSetting
const directory = dirname(reportPath)
const workspace = resolve(import.meta.dirname, '..')
const databaseHost = databaseUrl.hostname.replace(/^\[|\]$/g, '')
await mkdir(directory, { recursive: true })
const searchProxy = chunksOnly ? undefined : await createSearchProxy()
const sql = postgres(searchProxy?.url ?? databaseUrl.href, { max: 1 })
const results: Array<{
name: string
status: 'passed' | 'failed'
durationMs: number
error?: string
}> = []
const vector = JSON.stringify([1, ...Array.from({ length: 1535 }, () => 0)])
let created = false
const preloadPath = join(directory, 'embedding-transport.mjs')
type ProviderScenario =
| 'success'
| '429'
| '503'
| '401'
| 'dimensions'
| 'indices'
| 'batch-failure'
| 'long-retry'
let providerScenario: ProviderScenario = 'success'
let scenarioRequests = 0
let provider: Server<undefined> | undefined
const providerRequests: Array<{
scenario: ProviderScenario
attempt: number
status: number
inputs: number
}> = []
async function check(name: string, action: () => Promise<void>) {
const started = performance.now()
try {
await action()
results.push({ name, status: 'passed', durationMs: Math.round(performance.now() - started) })
} catch (error) {
results.push({
name,
status: 'failed',
durationMs: Math.round(performance.now() - started),
error: getErrorMessage(error),
})
throw error
}
}
async function createSearchProxy() {
const sockets = new Set<Socket>()
const buffered = new Map<Socket, Buffer[]>()
let paused = false
const server = createServer((client) => {
const upstream = connect(Number(databaseUrl.port || 5432), databaseHost)
sockets.add(client)
sockets.add(upstream)
assert(sockets.size <= 16, 'Search proxy connection cap exceeded')
client.on('error', () => upstream.destroy())
upstream.on('error', () => client.destroy())
client.on('close', () => {
sockets.delete(client)
upstream.destroy()
})
upstream.on('close', () => {
sockets.delete(upstream)
buffered.delete(client)
client.destroy()
})
client.pipe(upstream)
upstream.on('data', (data: Buffer) => {
if (!paused) {
client.write(data)
return
}
const chunks = buffered.get(client) ?? []
assert(
chunks.reduce((bytes, chunk) => bytes + chunk.length, 0) + data.length <= 1024 * 1024,
'Search proxy byte cap exceeded'
)
chunks.push(data)
buffered.set(client, chunks)
})
})
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
const address = server.address()
assert(address && typeof address !== 'string')
const url = new URL(databaseUrl)
url.hostname = '127.0.0.1'
url.port = String(address.port)
return {
url: url.href,
pause() {
paused = true
},
resume() {
paused = false
for (const [client, chunks] of buffered) for (const chunk of chunks) client.write(chunk)
buffered.clear()
},
async close() {
for (const socket of sockets) socket.destroy()
await new Promise<void>((resolve) => server.close(() => resolve()))
},
}
}
async function within<T>(operation: Promise<T>, milliseconds: number): Promise<T> {
return Promise.race([
operation,
sleep(milliseconds).then(() => {
throw new Error('Search exceeded its response budget')
}),
])
}
async function index(
mode: 'check' | 'publish',
name: string,
paths: string[] = [],
useProvider = false
) {
const outputPath = join(directory, `${name}.json`)
const child = spawn(
process.execPath,
[
'--no-env-file',
...(useProvider ? ['--preload', preloadPath] : []),
'scripts/index-search.ts',
...(mode === 'check' ? ['--check'] : []),
...paths.map((path) => `--path=${path}`),
],
{
cwd: workspace,
env: {
...process.env,
DATABASE_URL: searchProxy?.url ?? databaseUrl.href,
OPENAI_API_KEY: useProvider ? 'isolated-fixture' : '',
DOCS_SEARCH_REPORT_PATH: outputPath,
},
stdio: 'ignore',
timeout: 60000,
}
)
const exitCode = await new Promise<number>((resolve, reject) => {
child.once('error', reject)
child.once('close', (code) => resolve(code ?? 1))
})
return { exitCode, report: await readFile(outputPath, 'utf8') }
}
function scenario(value: ProviderScenario) {
providerScenario = value
scenarioRequests = 0
}
async function documentState(source: string) {
const [row] = await sql<Array<{ count: number; digest: string }>>`
SELECT count(*)::int AS count, md5(string_agg(row_to_json(document)::text, '|' ORDER BY chunk_id)) AS digest
FROM docs_embeddings AS document WHERE source_document = ${source}
`
assert(row)
return row
}
async function fingerprint() {
const [row] = await sql<Array<{ count: number; digest: string }>>`
SELECT count(*)::int AS count, md5(string_agg(chunk_id::text, ',' ORDER BY chunk_id)) AS digest FROM docs_embeddings
`
assert(row)
return row
}
async function verifyChunks() {
const parser = unified().use(remarkParse).use(remarkGfm)
for (const path of [
'/api-reference/access-requests/cancelOrganizationAccessRequest',
'/integrations/buffer',
'/integrations/plane',
]) {
const page = getPage(path)
assert(page)
const source = parser.parse(page.markdown)
const chunks = [...pageSearchChunks(page)]
const trees = chunks.map((chunk) => {
assert(chunk.text.length <= 6000, `${chunk.id}: chunk exceeds the complete payload budget`)
assert.equal(new URL(chunk.sourceLink).pathname, path)
const anchor = new URL(chunk.sourceLink).hash.slice(1)
assert(
!anchor || page.headings.some((heading) => heading.id === anchor),
`${chunk.id}: unknown citation`
)
if (!chunk.id.endsWith(':0')) {
assert(
chunk.text.includes(page.title) &&
chunk.text.includes(chunk.headerText) &&
/continued/i.test(chunk.text),
`${chunk.id}: continuation lost its page/section context`
)
}
const tree = parser.parse(chunk.text)
visit(tree, 'code', (node) => {
const raw = chunk.text.slice(node.position?.start.offset, node.position?.end.offset)
const marker = /^ {0,3}(`{3,}|~{3,})/.exec(raw)?.[1]
if (marker)
assert(
raw.trimEnd().split('\n').at(-1)?.trim().startsWith(marker),
`${chunk.id}: unclosed fenced code`
)
})
return tree
})
const originalCode: Array<{ value: string; lang: string | null | undefined }> = []
visit(source, 'code', (node) => {
originalCode.push({ value: node.value, lang: node.lang })
})
let sourceIndex = 0
let reconstructed = ''
let pieces = 0
for (const tree of trees)
visit(tree, 'code', (node) => {
const expected = originalCode[sourceIndex]
assert(expected, `${path}: unexpected code block`)
assert.equal(node.lang, expected.lang, `${path}: code language changed`)
reconstructed += `${pieces++ ? '\n' : ''}${node.value}`
assert(expected.value.startsWith(reconstructed), `${path}: code bytes or order changed`)
if (reconstructed === expected.value) {
sourceIndex += 1
reconstructed = ''
pieces = 0
}
})
assert.equal(sourceIndex, originalCode.length, `${path}: code content was lost`)
assert.equal(pieces, 0, `${path}: code continuation was incomplete`)
visit(source, 'paragraph', (node) => {
const raw = page.markdown.slice(node.position?.start.offset, node.position?.end.offset)
if (raw.length < 4500)
assert(
chunks.some((chunk) => chunk.text.includes(raw)),
`${path}: a fitting paragraph was split`
)
})
const emittedRows: string[][] = []
for (const tree of trees)
visit(tree, 'tableRow', (row) => {
emittedRows.push(row.children.map((cell) => markdownToString(cell)))
})
visit(source, 'table', (table) => {
const header = page.markdown.slice(
table.children[0]?.position?.start.offset,
table.children[0]?.position?.end.offset
)
for (const row of table.children.slice(1)) {
const raw = page.markdown.slice(row.position?.start.offset, row.position?.end.offset)
if (raw.length < 4500) {
const containing = chunks.find((chunk) => chunk.text.includes(raw))
assert(
containing?.text.includes(header),
`${path}: fitting table row or repeated header lost`
)
} else {
const cells = row.children.map((cell) => markdownToString(cell))
const matching = emittedRows.filter((candidate) => candidate[0] === cells[0])
for (const [index, cell] of cells.entries()) {
const retained = matching
.map((candidate) => candidate[index] ?? '')
.join(' ')
.replace(/\s+/g, ' ')
assert(
retained.includes(cell.replace(/\s+/g, ' ')),
`${path}: oversized table cell lost content`
)
}
}
}
})
}
}
async function verifyDatabase() {
const [existing] = await sql<
Array<{ count: number }>
>`SELECT count(*)::int AS count FROM information_schema.tables WHERE table_schema = 'public'`
assert(existing?.count === 0, 'Refusing to mutate a database that already contains tables')
const proxy = searchProxy
assert(proxy, 'Search proxy fixture is required for database verification')
process.env.DATABASE_URL = proxy.url
await sql`CREATE EXTENSION IF NOT EXISTS vector`
await sql`
CREATE TABLE docs_embeddings (
chunk_id uuid PRIMARY KEY,
chunk_text text NOT NULL,
source_document text NOT NULL,
source_link text NOT NULL,
header_text text NOT NULL,
header_level integer NOT NULL,
token_count integer NOT NULL,
embedding vector(1536) NOT NULL,
embedding_model text NOT NULL DEFAULT 'text-embedding-3-small',
metadata jsonb NOT NULL DEFAULT '{}',
chunk_text_tsv tsvector GENERATED ALWAYS AS (to_tsvector('english', chunk_text)) STORED,
created_at timestamp NOT NULL DEFAULT now(),
updated_at timestamp NOT NULL DEFAULT now()
)
`
created = true
await sql`CREATE INDEX ON docs_embeddings(source_document)`
await sql`CREATE INDEX ON docs_embeddings USING gin(chunk_text_tsv)`
const page = getPage('/introduction')
assert(page)
const probe = generateId()
const query = 'simhybridprobe'
await sql`
INSERT INTO docs_embeddings (chunk_id, chunk_text, source_document, source_link, header_text, header_level, token_count, embedding, metadata)
VALUES (${probe}, ${query}, ${page.sourcePath}, 'https://docs.sim.ai/introduction', 'Introduction', 1, 1, ${vector}::vector, ${sql.json({ contentHash: page.contentHash })})
`
await check('Real PostgreSQL keyword results work with no embedding key', async () => {
assert.equal(process.env.OPENAI_API_KEY, '')
const stored = await searchDatabase()<
Array<{ chunk_id: string; metadata: { contentHash: string } }>
>`SELECT chunk_id, metadata FROM docs_embeddings WHERE chunk_text_tsv @@ plainto_tsquery('english', ${query})`
assert.equal(stored[0]?.chunk_id, probe)
assert.equal(stored[0]?.metadata.contentHash, page.contentHash)
const results = await searchDocs(query)
assert.equal(results.length, 1)
assert.equal(results[0]?.id, probe)
assert.equal(results[0]?.url, 'https://docs.sim.ai/introduction')
})
await check('Local search matches do not wait for database traffic', async () => {
proxy.pause()
const pending = searchDocs('Slack')
try {
const results = await within(pending, 500)
assert(results.some((result) => new URL(result.url).pathname === '/integrations/slack'))
} finally {
proxy.resume()
await pending
}
})
await check(
'Hybrid deadlines keep stalled requests bounded and recover after the network resumes',
async () => {
proxy.pause()
try {
assert.deepEqual(await within(searchDocs(query), 2000), [])
assert.deepEqual(await within(searchDocs(query), 2000), [])
assert.deepEqual(await within(searchDocs(query), 500), [])
} finally {
proxy.resume()
await searchDatabase()`SELECT 1`
}
assert.equal((await searchDocs(query))[0]?.id, probe)
}
)
await check('Database keywords survive a stalled embedding provider', async () => {
const transport = globalThis.fetch
let requests = 0
const delayedProvider = serve({
hostname: '127.0.0.1',
port: 0,
async fetch() {
assert(++requests <= 1, 'Search embedding request cap exceeded')
await sleep(3000)
return Response.json({ data: [{ embedding: JSON.parse(vector) }] })
},
})
globalThis.fetch = Object.assign(
(...[input, init]: Parameters<typeof fetch>) =>
transport(
input === 'https://api.openai.com/v1/embeddings' ? delayedProvider.url : input,
init
),
transport
)
process.env.OPENAI_API_KEY = 'isolated-fixture'
try {
assert.equal((await within(searchDocs(query), 2000))[0]?.id, probe)
assert.equal(requests, 1)
} finally {
globalThis.fetch = transport
process.env.OPENAI_API_KEY = ''
await delayedProvider.stop(true)
}
})
await check('Repeated lower-ranked chunks do not outrank a stronger page match', async () => {
const otherPage = getPage('/getting-started')
assert(otherPage)
const rankingQuery = 'simrankingprobe'
try {
for (const candidate of [page, otherPage])
await sql`
INSERT INTO docs_embeddings (chunk_id, chunk_text, source_document, source_link, header_text, header_level, token_count, embedding, metadata)
VALUES (${generateId()}, ${candidate === page ? `${rankingQuery} ${rankingQuery}` : rankingQuery}, ${candidate.sourcePath}, ${`https://docs.sim.ai${candidate.url}`}, ${candidate.title}, 1, 2, ${vector}::vector, ${sql.json({ contentHash: candidate.contentHash })})
`
const expected = ['https://docs.sim.ai/introduction', 'https://docs.sim.ai/getting-started']
assert.deepEqual(
(await searchDocs(rankingQuery)).map((result) => result.url),
expected
)
await sql`
INSERT INTO docs_embeddings (chunk_id, chunk_text, source_document, source_link, header_text, header_level, token_count, embedding, metadata)
VALUES (${generateId()}, ${rankingQuery}, ${otherPage.sourcePath}, 'https://docs.sim.ai/getting-started', ${otherPage.title}, 1, 1, ${vector}::vector, ${sql.json({ contentHash: otherPage.contentHash })})
`
assert.deepEqual(
(await searchDocs(rankingQuery)).map((result) => result.url),
expected
)
} finally {
await sql`DELETE FROM docs_embeddings WHERE chunk_text_tsv @@ plainto_tsquery('english', ${rankingQuery})`
}
})
await check('Stale index hashes cannot supply search citations', async () => {
await sql`UPDATE docs_embeddings SET metadata = '{"contentHash":"stale"}' WHERE chunk_id = ${probe}`
assert.deepEqual(await searchDocs(query), [])
await sql`UPDATE docs_embeddings SET metadata = ${sql.json({ contentHash: page.contentHash })} WHERE chunk_id = ${probe}`
})
await check(
'Foreign origins and nonexistent anchors cannot supply search citations',
async () => {
for (const url of [
'https://untrusted.example/introduction',
'https://docs.sim.ai/introduction#unknown-anchor',
]) {
await sql`UPDATE docs_embeddings SET source_link = ${url} WHERE chunk_id = ${probe}`
assert.deepEqual(await searchDocs(query), [])
}
await sql`UPDATE docs_embeddings SET source_link = 'https://docs.sim.ai/introduction' WHERE chunk_id = ${probe}`
}
)
await check(
'Database query failures return no matches and recover after restoration',
async () => {
await sql`ALTER TABLE docs_embeddings RENAME TO docs_embeddings_unavailable`
try {
assert.deepEqual(await searchDocs(query), [])
} finally {
await sql`ALTER TABLE docs_embeddings_unavailable RENAME TO docs_embeddings`
}
assert.equal((await searchDocs(query))[0]?.id, probe)
}
)
await sql`DELETE FROM docs_embeddings`
const pages = getPages().filter((page) => page.url !== '/')
for (const page of pages) {
const chunks = [...pageSearchChunks(page)]
const source = chunks[0]?.sourceDocument
assert(source)
const rows = chunks.map((chunk) => ({
chunk_id: generateId(),
chunk_text: chunk.text,
source_link: chunk.sourceLink,
header_text: chunk.headerText,
header_level: chunk.headerLevel,
token_count: Math.ceil(chunk.text.length / 4),
metadata: {
title: page.title,
contentHash: page.contentHash,
chunkId: chunk.id,
embeddingInputHash: createHash('sha256')
.update(
JSON.stringify([
'text-embedding-3-small',
1536,
`${page.title}\n${chunk.headerText}\n${chunk.text}`,
])
)
.digest('hex'),
catalog: 'sim-docs-v2',
},
}))
await sql`
INSERT INTO docs_embeddings (chunk_id, chunk_text, source_document, source_link, header_text, header_level, token_count, embedding, metadata)
SELECT chunk_id, chunk_text, ${source}, source_link, header_text, header_level, token_count, ${vector}::vector, metadata
FROM jsonb_to_recordset(${sql.json(rows)}) AS fixture(chunk_id uuid, chunk_text text, source_link text, header_text text, header_level integer, token_count integer, metadata jsonb)
`
}
const fresh = await fingerprint()
await check(
'Fresh index checks succeed and repeated publication preserves row identities without credentials',
async () => {
assert.equal((await index('check', 'fresh-check')).exitCode, 0)
assert.equal((await index('publish', 'fresh-publish')).exitCode, 0)
assert.deepEqual(await fingerprint(), fresh)
}
)
await check(
'Citation-only changes are stale and repair without replacing embeddings',
async () => {
const citations = await sql<
Array<{ chunk_id: string; source_link: string; embedding: string }>
>`SELECT chunk_id, source_link, embedding::text FROM docs_embeddings WHERE source_document = ${page.sourcePath} ORDER BY chunk_id`
assert(citations.length > 0)
await sql`UPDATE docs_embeddings SET source_link = 'https://old.example/introduction' WHERE source_document = ${page.sourcePath}`
assert.equal((await index('check', 'citation-stale', ['/introduction'])).exitCode, 1)
const repaired = await index('publish', 'citation-repaired', ['/introduction'])
assert.equal(repaired.exitCode, 0)
assert.deepEqual(await fingerprint(), fresh)
const repairedCitations = await sql<
Array<{ chunk_id: string; source_link: string; embedding: string }>
>`SELECT chunk_id, source_link, embedding::text FROM docs_embeddings WHERE source_document = ${page.sourcePath} ORDER BY chunk_id`
assert.deepEqual(repairedCitations, citations)
assert.equal((await index('check', 'citation-fresh', ['/introduction'])).exitCode, 0)
}
)
await check('Concurrent publishers fail before modifying any rows', async () => {
await sql`SELECT pg_advisory_lock(724130561)`
try {
const result = await index('publish', 'concurrent-publish', ['/introduction'])
assert.equal(result.exitCode, 1)
assert(result.report.includes('Another documentation index publisher or check is active'))
assert.deepEqual(await fingerprint(), fresh)
} finally {
await sql`SELECT pg_advisory_unlock(724130561)`
}
})
await check('Freshness rejects modified chunk bytes and embedding model identities', async () => {
const [row] = await sql<
Array<{ chunk_id: string; chunk_text: string }>
>`SELECT chunk_id, chunk_text FROM docs_embeddings WHERE source_document = ${page.sourcePath} LIMIT 1`
assert(row)
await sql`UPDATE docs_embeddings SET chunk_text = 'corrupt stored chunk' WHERE chunk_id = ${row.chunk_id}`
assert.equal((await index('check', 'corrupt-text', ['/introduction'])).exitCode, 1)
const failure = await index('publish', 'corrupt-no-key', ['/introduction'])
assert.equal(failure.exitCode, 1)
assert(failure.report.includes('OPENAI_API_KEY is required'))
assert.deepEqual(await fingerprint(), fresh)
await sql`UPDATE docs_embeddings SET chunk_text = ${row.chunk_text}, embedding_model = 'wrong-model' WHERE chunk_id = ${row.chunk_id}`
assert.equal((await index('check', 'wrong-model', ['/introduction'])).exitCode, 1)
await sql`UPDATE docs_embeddings SET embedding_model = 'text-embedding-3-small' WHERE chunk_id = ${row.chunk_id}`
assert.equal((await index('check', 'restored-chunk', ['/introduction'])).exitCode, 0)
})
await sql`
INSERT INTO docs_embeddings (chunk_id, chunk_text, source_document, source_link, header_text, header_level, token_count, embedding, metadata)
VALUES (${generateId()}, 'retired fixture', 'retired-fixture.mdx', 'https://docs.sim.ai/retired-fixture', 'Retired', 1, 1, ${vector}::vector, '{}')
`
await check(
'Freshness checks reject retired pages and publication retires only those rows',
async () => {
const check = await index('check', 'retired-check')
assert.equal(check.exitCode, 1)
assert(check.report.includes('removed documents remain indexed'))
assert.equal((await index('publish', 'retired-publish')).exitCode, 0)
assert.deepEqual(await fingerprint(), fresh)
}
)
const first = pages[0]
assert(first)
const source = [...pageSearchChunks(first)][0]?.sourceDocument
assert(source)
await sql`UPDATE docs_embeddings SET metadata = '{"contentHash":"stale"}' WHERE source_document = ${source}`
await check(
'A missing embedding key fails publication without deleting old document data',
async () => {
const before = await fingerprint()
const result = await index('publish', 'failed-publish')
assert.equal(result.exitCode, 1)
assert(result.report.includes('OPENAI_API_KEY is required'))
assert.deepEqual(await fingerprint(), before)
assert.equal((await index('check', 'stale-check')).exitCode, 1)
}
)
const responseValues = new Map<string, number>()
const requestSchema = z.object({
input: z.array(z.string().min(1).max(8000)).min(1).max(16),
model: z.literal('text-embedding-3-small'),
dimensions: z.literal(1536),
encoding_format: z.literal('float'),
})
provider = serve({
hostname: '127.0.0.1',
port: 0,
async fetch(request) {
assert.equal(new URL(request.url).pathname, '/v1/embeddings')
assert.equal(request.method, 'POST')
assert.equal(request.headers.get('authorization'), 'Bearer isolated-fixture')
const body = requestSchema.parse(await request.json())
scenarioRequests += 1
assert(providerRequests.length < 16, 'Provider fixture request cap exceeded')
const status =
providerScenario === '401'
? 401
: providerScenario === 'long-retry'
? 429
: providerScenario === 'batch-failure' && scenarioRequests > 1
? 503
: (providerScenario === '429' || providerScenario === '503') && scenarioRequests === 1
? Number(providerScenario)
: 200
providerRequests.push({
scenario: providerScenario,
attempt: scenarioRequests,
status,
inputs: body.input.length,
})
if (status !== 200)
return Response.json(
{ error: { message: 'Isolated fixture failure' } },
{ status, headers: { 'Retry-After': providerScenario === 'long-retry' ? '120' : '0' } }
)
const data = body.input
.map((input, index) => {
responseValues.set(input, index + 1)
return {
index: providerScenario === 'indices' ? body.input.length : index,
embedding: [
index + 1,
...Array.from({ length: providerScenario === 'dimensions' ? 1534 : 1535 }, () => 0),
],
}
})
.reverse()
return Response.json({ data, usage: { total_tokens: body.input.length } })
},
})
await writeFile(
preloadPath,
`
const endpoint = ${JSON.stringify(new URL('/v1/embeddings', provider.url).href)};
const transport = globalThis.fetch;
globalThis.fetch = Object.assign(async (input, init) => {
const request = new Request(input, init);
if (request.url !== 'https://api.openai.com/v1/embeddings') throw new Error('Unexpected provider URL in isolated test');
return transport(new Request(endpoint, request));
}, transport);
`
)
const introChunks = [...pageSearchChunks(page)]
assert(
introChunks.length > 0 && introChunks.length <= 16,
'Introduction must fit one bounded fixture batch'
)
await check(
'First publication writes provider vectors in input order and an unchanged repeat makes no requests',
async () => {
await sql`DELETE FROM docs_embeddings WHERE source_document = ${page.sourcePath}`
scenario('success')
assert.equal((await index('publish', 'provider-first', ['/introduction'], true)).exitCode, 0)
assert.equal(scenarioRequests, 1)
const rows = await sql<
Array<{
chunk_text: string
header_text: string
source_link: string
embedding: string
metadata: { title: string }
}>
>`
SELECT chunk_text, header_text, source_link, embedding::text, metadata FROM docs_embeddings WHERE source_document = ${page.sourcePath}
`
assert.equal(rows.length, providerRequests.at(-1)?.inputs)
for (const row of rows) {
const input = `${row.metadata.title}\n${row.header_text}\n${row.chunk_text}`
const values = z.array(z.number()).parse(JSON.parse(row.embedding))
assert.equal(values.length, 1536)
assert.equal(values[0], responseValues.get(input))
assert.equal(new URL(row.source_link).pathname, '/introduction')
}
const before = await documentState(page.sourcePath)
assert.equal(
(await index('publish', 'provider-unchanged', ['/introduction'], true)).exitCode,
0
)
assert.equal(scenarioRequests, 1)
assert.deepEqual(await documentState(page.sourcePath), before)
}
)
await check(
'Changed stored chunk bytes are replaced while retaining other row identities',
async () => {
const rows = await sql<
Array<{ chunk_id: string }>
>`SELECT chunk_id FROM docs_embeddings WHERE source_document = ${page.sourcePath} ORDER BY chunk_id`
const changed = rows[0]
assert(changed)
await sql`UPDATE docs_embeddings SET chunk_text = 'stale provider fixture' WHERE chunk_id = ${changed.chunk_id}`
scenario('success')
assert.equal(
(await index('publish', 'provider-changed', ['/introduction'], true)).exitCode,
0
)
assert.equal(scenarioRequests, 1)
assert.equal(providerRequests.at(-1)?.inputs, 1)
const after = await sql<
Array<{ chunk_id: string }>
>`SELECT chunk_id FROM docs_embeddings WHERE source_document = ${page.sourcePath}`
assert.equal(after.length, rows.length)
assert(!after.some((row) => row.chunk_id === changed.chunk_id))
for (const retained of rows.slice(1))
assert(after.some((row) => row.chunk_id === retained.chunk_id))
}
)
for (const status of ['429', '503'] as const) {
await check(
`Provider HTTP ${status} retries once and publishes the successful response`,
async () => {
await sql`UPDATE docs_embeddings SET chunk_text = 'stale provider fixture' WHERE chunk_id = (SELECT chunk_id FROM docs_embeddings WHERE source_document = ${page.sourcePath} LIMIT 1)`
scenario(status)
assert.equal(
(await index('publish', `provider-retry-${status}`, ['/introduction'], true)).exitCode,
0
)
assert.equal(scenarioRequests, 2)
assert.equal(providerRequests.at(-2)?.status, Number(status))
assert.equal(providerRequests.at(-1)?.status, 200)
assert.equal(
(await index('check', `provider-fresh-${status}`, ['/introduction'])).exitCode,
0
)
}
)
}
for (const failure of ['401', 'dimensions', 'indices', 'long-retry'] as const) {
await check(
`Provider ${failure} fails without retry or modifying the previous document`,
async () => {
await sql`UPDATE docs_embeddings SET chunk_text = 'stale provider fixture' WHERE chunk_id = (SELECT chunk_id FROM docs_embeddings WHERE source_document = ${page.sourcePath} LIMIT 1)`
const before = await documentState(page.sourcePath)
scenario(failure)
const result = await index(
'publish',
`provider-failure-${failure}`,
['/introduction'],
true
)
assert.equal(result.exitCode, 1)
assert.equal(scenarioRequests, 1)
assert.deepEqual(await documentState(page.sourcePath), before)
}
)
}
await check(
'A successful first batch followed by persistent failure leaves the entire previous document intact',
async () => {
const multiBatch = pages.find((candidate) => {
const count = [...pageSearchChunks(candidate)].length
return count > 16 && count <= 32
})
assert(multiBatch, 'A bounded two-batch document is required')
const source = [...pageSearchChunks(multiBatch)][0]?.sourceDocument
assert(source)
await sql`UPDATE docs_embeddings SET chunk_text = 'stale batch fixture' WHERE source_document = ${source}`
const before = await documentState(source)
scenario('batch-failure')
const result = await index('publish', 'provider-partial-failure', [multiBatch.url], true)
assert.equal(result.exitCode, 1)
assert(scenarioRequests >= 2 && scenarioRequests <= 4)
const attempts = providerRequests.filter((row) => row.scenario === 'batch-failure')
assert.equal(attempts[0]?.status, 200)
assert.equal(attempts[0]?.inputs, 16)
assert(attempts.slice(1).every((row) => row.status === 503))
assert.deepEqual(await documentState(source), before)
}
)
}
try {
await check(
'Compiled publication chunks preserve code, tables, context and bounded payloads',
verifyChunks
)
if (!chunksOnly) await verifyDatabase()
} catch (error) {
if (!results.some((result) => result.status === 'failed')) {
results.push({
name: 'Fixture lifecycle',
status: 'failed',
durationMs: 0,
error: getErrorMessage(error),
})
}
} finally {
searchProxy?.resume()
await provider?.stop(true)
await rm(preloadPath, { force: true })
if (created) await sql`DROP TABLE docs_embeddings`
await Promise.all([sql.end({ timeout: 5 }), searchDatabase().end({ timeout: 5 })])
await searchProxy?.close()
await writeFile(
reportPath,
`${JSON.stringify({ scope: chunksOnly ? 'compiled-content' : 'postgres-publication', results, provider: { mode: 'loopback-http-fixture', maxRequests: 16, maxInputsPerRequest: 16, requests: providerRequests } }, null, 2)}\n`
)
}
logger.info(
chunksOnly ? 'Verified compiled search chunks' : 'Verified isolated PostgreSQL search boundary',
{ checks: results.length, reportPath }
)
if (results.some((result) => result.status === 'failed')) process.exitCode = 1