import assert from 'node:assert/strict' import { spawn } from 'node:child_process' import { createHash } from 'node:crypto' import { mkdir, readFile, rm, writeFile } from 'node:fs/promises' import { connect, createServer, type Socket } from 'node:net' import { dirname, join, resolve } from 'node:path' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' import { type Server, serve } from 'bun' import { toString as markdownToString } from 'mdast-util-to-string' import postgres from 'postgres' import remarkGfm from 'remark-gfm' import remarkParse from 'remark-parse' import { unified } from 'unified' import { visit } from 'unist-util-visit' import { z } from 'zod' import { getPage, getPages } from '@/lib/content/catalog' import { pageSearchChunks } from '@/lib/search/chunks' import { searchDatabase } from '@/lib/search/database' import { searchDocs } from '@/lib/search/service' const logger = createLogger('DocsSearchPostgresE2E') const chunksOnly = process.argv.includes('--chunks-only') const databaseSetting = process.env.DOCS_SEARCH_E2E_DATABASE_URL ?? (chunksOnly ? 'postgres://127.0.0.1:1/docs_search_test' : undefined) const reportSetting = process.env.DOCS_SEARCH_E2E_REPORT_PATH assert( databaseSetting && reportSetting, 'Explicit disposable database and report paths are required' ) const databaseUrl = new URL(databaseSetting) assert( ['localhost', '127.0.0.1', '[::1]'].includes(databaseUrl.hostname), 'Only loopback databases are permitted' ) assert(databaseUrl.pathname === '/docs_search_test', 'Use a dedicated docs_search_test database') assert(['postgres:', 'postgresql:'].includes(databaseUrl.protocol), 'A PostgreSQL URL is required') process.env.DATABASE_URL = databaseUrl.href process.env.OPENAI_API_KEY = '' const reportPath: string = reportSetting const directory = dirname(reportPath) const workspace = resolve(import.meta.dirname, '..') const databaseHost = databaseUrl.hostname.replace(/^\[|\]$/g, '') await mkdir(directory, { recursive: true }) const searchProxy = chunksOnly ? undefined : await createSearchProxy() const sql = postgres(searchProxy?.url ?? databaseUrl.href, { max: 1 }) const results: Array<{ name: string status: 'passed' | 'failed' durationMs: number error?: string }> = [] const vector = JSON.stringify([1, ...Array.from({ length: 1535 }, () => 0)]) let created = false const preloadPath = join(directory, 'embedding-transport.mjs') type ProviderScenario = | 'success' | '429' | '503' | '401' | 'dimensions' | 'indices' | 'batch-failure' | 'long-retry' let providerScenario: ProviderScenario = 'success' let scenarioRequests = 0 let provider: Server | undefined const providerRequests: Array<{ scenario: ProviderScenario attempt: number status: number inputs: number }> = [] async function check(name: string, action: () => Promise) { const started = performance.now() try { await action() results.push({ name, status: 'passed', durationMs: Math.round(performance.now() - started) }) } catch (error) { results.push({ name, status: 'failed', durationMs: Math.round(performance.now() - started), error: getErrorMessage(error), }) throw error } } async function createSearchProxy() { const sockets = new Set() const buffered = new Map() let paused = false const server = createServer((client) => { const upstream = connect(Number(databaseUrl.port || 5432), databaseHost) sockets.add(client) sockets.add(upstream) assert(sockets.size <= 16, 'Search proxy connection cap exceeded') client.on('error', () => upstream.destroy()) upstream.on('error', () => client.destroy()) client.on('close', () => { sockets.delete(client) upstream.destroy() }) upstream.on('close', () => { sockets.delete(upstream) buffered.delete(client) client.destroy() }) client.pipe(upstream) upstream.on('data', (data: Buffer) => { if (!paused) { client.write(data) return } const chunks = buffered.get(client) ?? [] assert( chunks.reduce((bytes, chunk) => bytes + chunk.length, 0) + data.length <= 1024 * 1024, 'Search proxy byte cap exceeded' ) chunks.push(data) buffered.set(client, chunks) }) }) await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) const address = server.address() assert(address && typeof address !== 'string') const url = new URL(databaseUrl) url.hostname = '127.0.0.1' url.port = String(address.port) return { url: url.href, pause() { paused = true }, resume() { paused = false for (const [client, chunks] of buffered) for (const chunk of chunks) client.write(chunk) buffered.clear() }, async close() { for (const socket of sockets) socket.destroy() await new Promise((resolve) => server.close(() => resolve())) }, } } async function within(operation: Promise, milliseconds: number): Promise { return Promise.race([ operation, sleep(milliseconds).then(() => { throw new Error('Search exceeded its response budget') }), ]) } async function index( mode: 'check' | 'publish', name: string, paths: string[] = [], useProvider = false ) { const outputPath = join(directory, `${name}.json`) const child = spawn( process.execPath, [ '--no-env-file', ...(useProvider ? ['--preload', preloadPath] : []), 'scripts/index-search.ts', ...(mode === 'check' ? ['--check'] : []), ...paths.map((path) => `--path=${path}`), ], { cwd: workspace, env: { ...process.env, DATABASE_URL: searchProxy?.url ?? databaseUrl.href, OPENAI_API_KEY: useProvider ? 'isolated-fixture' : '', DOCS_SEARCH_REPORT_PATH: outputPath, }, stdio: 'ignore', timeout: 60000, } ) const exitCode = await new Promise((resolve, reject) => { child.once('error', reject) child.once('close', (code) => resolve(code ?? 1)) }) return { exitCode, report: await readFile(outputPath, 'utf8') } } function scenario(value: ProviderScenario) { providerScenario = value scenarioRequests = 0 } async function documentState(source: string) { const [row] = await sql>` SELECT count(*)::int AS count, md5(string_agg(row_to_json(document)::text, '|' ORDER BY chunk_id)) AS digest FROM docs_embeddings AS document WHERE source_document = ${source} ` assert(row) return row } async function fingerprint() { const [row] = await sql>` SELECT count(*)::int AS count, md5(string_agg(chunk_id::text, ',' ORDER BY chunk_id)) AS digest FROM docs_embeddings ` assert(row) return row } async function verifyChunks() { const parser = unified().use(remarkParse).use(remarkGfm) for (const path of [ '/api-reference/access-requests/cancelOrganizationAccessRequest', '/integrations/buffer', '/integrations/plane', ]) { const page = getPage(path) assert(page) const source = parser.parse(page.markdown) const chunks = [...pageSearchChunks(page)] const trees = chunks.map((chunk) => { assert(chunk.text.length <= 6000, `${chunk.id}: chunk exceeds the complete payload budget`) assert.equal(new URL(chunk.sourceLink).pathname, path) const anchor = new URL(chunk.sourceLink).hash.slice(1) assert( !anchor || page.headings.some((heading) => heading.id === anchor), `${chunk.id}: unknown citation` ) if (!chunk.id.endsWith(':0')) { assert( chunk.text.includes(page.title) && chunk.text.includes(chunk.headerText) && /continued/i.test(chunk.text), `${chunk.id}: continuation lost its page/section context` ) } const tree = parser.parse(chunk.text) visit(tree, 'code', (node) => { const raw = chunk.text.slice(node.position?.start.offset, node.position?.end.offset) const marker = /^ {0,3}(`{3,}|~{3,})/.exec(raw)?.[1] if (marker) assert( raw.trimEnd().split('\n').at(-1)?.trim().startsWith(marker), `${chunk.id}: unclosed fenced code` ) }) return tree }) const originalCode: Array<{ value: string; lang: string | null | undefined }> = [] visit(source, 'code', (node) => { originalCode.push({ value: node.value, lang: node.lang }) }) let sourceIndex = 0 let reconstructed = '' let pieces = 0 for (const tree of trees) visit(tree, 'code', (node) => { const expected = originalCode[sourceIndex] assert(expected, `${path}: unexpected code block`) assert.equal(node.lang, expected.lang, `${path}: code language changed`) reconstructed += `${pieces++ ? '\n' : ''}${node.value}` assert(expected.value.startsWith(reconstructed), `${path}: code bytes or order changed`) if (reconstructed === expected.value) { sourceIndex += 1 reconstructed = '' pieces = 0 } }) assert.equal(sourceIndex, originalCode.length, `${path}: code content was lost`) assert.equal(pieces, 0, `${path}: code continuation was incomplete`) visit(source, 'paragraph', (node) => { const raw = page.markdown.slice(node.position?.start.offset, node.position?.end.offset) if (raw.length < 4500) assert( chunks.some((chunk) => chunk.text.includes(raw)), `${path}: a fitting paragraph was split` ) }) const emittedRows: string[][] = [] for (const tree of trees) visit(tree, 'tableRow', (row) => { emittedRows.push(row.children.map((cell) => markdownToString(cell))) }) visit(source, 'table', (table) => { const header = page.markdown.slice( table.children[0]?.position?.start.offset, table.children[0]?.position?.end.offset ) for (const row of table.children.slice(1)) { const raw = page.markdown.slice(row.position?.start.offset, row.position?.end.offset) if (raw.length < 4500) { const containing = chunks.find((chunk) => chunk.text.includes(raw)) assert( containing?.text.includes(header), `${path}: fitting table row or repeated header lost` ) } else { const cells = row.children.map((cell) => markdownToString(cell)) const matching = emittedRows.filter((candidate) => candidate[0] === cells[0]) for (const [index, cell] of cells.entries()) { const retained = matching .map((candidate) => candidate[index] ?? '') .join(' ') .replace(/\s+/g, ' ') assert( retained.includes(cell.replace(/\s+/g, ' ')), `${path}: oversized table cell lost content` ) } } } }) } } async function verifyDatabase() { const [existing] = await sql< Array<{ count: number }> >`SELECT count(*)::int AS count FROM information_schema.tables WHERE table_schema = 'public'` assert(existing?.count === 0, 'Refusing to mutate a database that already contains tables') const proxy = searchProxy assert(proxy, 'Search proxy fixture is required for database verification') process.env.DATABASE_URL = proxy.url await sql`CREATE EXTENSION IF NOT EXISTS vector` await sql` CREATE TABLE docs_embeddings ( chunk_id uuid PRIMARY KEY, chunk_text text NOT NULL, source_document text NOT NULL, source_link text NOT NULL, header_text text NOT NULL, header_level integer NOT NULL, token_count integer NOT NULL, embedding vector(1536) NOT NULL, embedding_model text NOT NULL DEFAULT 'text-embedding-3-small', metadata jsonb NOT NULL DEFAULT '{}', chunk_text_tsv tsvector GENERATED ALWAYS AS (to_tsvector('english', chunk_text)) STORED, created_at timestamp NOT NULL DEFAULT now(), updated_at timestamp NOT NULL DEFAULT now() ) ` created = true await sql`CREATE INDEX ON docs_embeddings(source_document)` await sql`CREATE INDEX ON docs_embeddings USING gin(chunk_text_tsv)` const page = getPage('/introduction') assert(page) const probe = generateId() const query = 'simhybridprobe' await sql` INSERT INTO docs_embeddings (chunk_id, chunk_text, source_document, source_link, header_text, header_level, token_count, embedding, metadata) VALUES (${probe}, ${query}, ${page.sourcePath}, 'https://docs.sim.ai/introduction', 'Introduction', 1, 1, ${vector}::vector, ${sql.json({ contentHash: page.contentHash })}) ` await check('Real PostgreSQL keyword results work with no embedding key', async () => { assert.equal(process.env.OPENAI_API_KEY, '') const stored = await searchDatabase()< Array<{ chunk_id: string; metadata: { contentHash: string } }> >`SELECT chunk_id, metadata FROM docs_embeddings WHERE chunk_text_tsv @@ plainto_tsquery('english', ${query})` assert.equal(stored[0]?.chunk_id, probe) assert.equal(stored[0]?.metadata.contentHash, page.contentHash) const results = await searchDocs(query) assert.equal(results.length, 1) assert.equal(results[0]?.id, probe) assert.equal(results[0]?.url, 'https://docs.sim.ai/introduction') }) await check('Local search matches do not wait for database traffic', async () => { proxy.pause() const pending = searchDocs('Slack') try { const results = await within(pending, 500) assert(results.some((result) => new URL(result.url).pathname === '/integrations/slack')) } finally { proxy.resume() await pending } }) await check( 'Hybrid deadlines keep stalled requests bounded and recover after the network resumes', async () => { proxy.pause() try { assert.deepEqual(await within(searchDocs(query), 2000), []) assert.deepEqual(await within(searchDocs(query), 2000), []) assert.deepEqual(await within(searchDocs(query), 500), []) } finally { proxy.resume() await searchDatabase()`SELECT 1` } assert.equal((await searchDocs(query))[0]?.id, probe) } ) await check('Database keywords survive a stalled embedding provider', async () => { const transport = globalThis.fetch let requests = 0 const delayedProvider = serve({ hostname: '127.0.0.1', port: 0, async fetch() { assert(++requests <= 1, 'Search embedding request cap exceeded') await sleep(3000) return Response.json({ data: [{ embedding: JSON.parse(vector) }] }) }, }) globalThis.fetch = Object.assign( (...[input, init]: Parameters) => transport( input === 'https://api.openai.com/v1/embeddings' ? delayedProvider.url : input, init ), transport ) process.env.OPENAI_API_KEY = 'isolated-fixture' try { assert.equal((await within(searchDocs(query), 2000))[0]?.id, probe) assert.equal(requests, 1) } finally { globalThis.fetch = transport process.env.OPENAI_API_KEY = '' await delayedProvider.stop(true) } }) await check('Repeated lower-ranked chunks do not outrank a stronger page match', async () => { const otherPage = getPage('/getting-started') assert(otherPage) const rankingQuery = 'simrankingprobe' try { for (const candidate of [page, otherPage]) await sql` INSERT INTO docs_embeddings (chunk_id, chunk_text, source_document, source_link, header_text, header_level, token_count, embedding, metadata) VALUES (${generateId()}, ${candidate === page ? `${rankingQuery} ${rankingQuery}` : rankingQuery}, ${candidate.sourcePath}, ${`https://docs.sim.ai${candidate.url}`}, ${candidate.title}, 1, 2, ${vector}::vector, ${sql.json({ contentHash: candidate.contentHash })}) ` const expected = ['https://docs.sim.ai/introduction', 'https://docs.sim.ai/getting-started'] assert.deepEqual( (await searchDocs(rankingQuery)).map((result) => result.url), expected ) await sql` INSERT INTO docs_embeddings (chunk_id, chunk_text, source_document, source_link, header_text, header_level, token_count, embedding, metadata) VALUES (${generateId()}, ${rankingQuery}, ${otherPage.sourcePath}, 'https://docs.sim.ai/getting-started', ${otherPage.title}, 1, 1, ${vector}::vector, ${sql.json({ contentHash: otherPage.contentHash })}) ` assert.deepEqual( (await searchDocs(rankingQuery)).map((result) => result.url), expected ) } finally { await sql`DELETE FROM docs_embeddings WHERE chunk_text_tsv @@ plainto_tsquery('english', ${rankingQuery})` } }) await check('Stale index hashes cannot supply search citations', async () => { await sql`UPDATE docs_embeddings SET metadata = '{"contentHash":"stale"}' WHERE chunk_id = ${probe}` assert.deepEqual(await searchDocs(query), []) await sql`UPDATE docs_embeddings SET metadata = ${sql.json({ contentHash: page.contentHash })} WHERE chunk_id = ${probe}` }) await check( 'Foreign origins and nonexistent anchors cannot supply search citations', async () => { for (const url of [ 'https://untrusted.example/introduction', 'https://docs.sim.ai/introduction#unknown-anchor', ]) { await sql`UPDATE docs_embeddings SET source_link = ${url} WHERE chunk_id = ${probe}` assert.deepEqual(await searchDocs(query), []) } await sql`UPDATE docs_embeddings SET source_link = 'https://docs.sim.ai/introduction' WHERE chunk_id = ${probe}` } ) await check( 'Database query failures return no matches and recover after restoration', async () => { await sql`ALTER TABLE docs_embeddings RENAME TO docs_embeddings_unavailable` try { assert.deepEqual(await searchDocs(query), []) } finally { await sql`ALTER TABLE docs_embeddings_unavailable RENAME TO docs_embeddings` } assert.equal((await searchDocs(query))[0]?.id, probe) } ) await sql`DELETE FROM docs_embeddings` const pages = getPages().filter((page) => page.url !== '/') for (const page of pages) { const chunks = [...pageSearchChunks(page)] const source = chunks[0]?.sourceDocument assert(source) const rows = chunks.map((chunk) => ({ chunk_id: generateId(), chunk_text: chunk.text, source_link: chunk.sourceLink, header_text: chunk.headerText, header_level: chunk.headerLevel, token_count: Math.ceil(chunk.text.length / 4), metadata: { title: page.title, contentHash: page.contentHash, chunkId: chunk.id, embeddingInputHash: createHash('sha256') .update( JSON.stringify([ 'text-embedding-3-small', 1536, `${page.title}\n${chunk.headerText}\n${chunk.text}`, ]) ) .digest('hex'), catalog: 'sim-docs-v2', }, })) await sql` INSERT INTO docs_embeddings (chunk_id, chunk_text, source_document, source_link, header_text, header_level, token_count, embedding, metadata) SELECT chunk_id, chunk_text, ${source}, source_link, header_text, header_level, token_count, ${vector}::vector, metadata FROM jsonb_to_recordset(${sql.json(rows)}) AS fixture(chunk_id uuid, chunk_text text, source_link text, header_text text, header_level integer, token_count integer, metadata jsonb) ` } const fresh = await fingerprint() await check( 'Fresh index checks succeed and repeated publication preserves row identities without credentials', async () => { assert.equal((await index('check', 'fresh-check')).exitCode, 0) assert.equal((await index('publish', 'fresh-publish')).exitCode, 0) assert.deepEqual(await fingerprint(), fresh) } ) await check( 'Citation-only changes are stale and repair without replacing embeddings', async () => { const citations = await sql< Array<{ chunk_id: string; source_link: string; embedding: string }> >`SELECT chunk_id, source_link, embedding::text FROM docs_embeddings WHERE source_document = ${page.sourcePath} ORDER BY chunk_id` assert(citations.length > 0) await sql`UPDATE docs_embeddings SET source_link = 'https://old.example/introduction' WHERE source_document = ${page.sourcePath}` assert.equal((await index('check', 'citation-stale', ['/introduction'])).exitCode, 1) const repaired = await index('publish', 'citation-repaired', ['/introduction']) assert.equal(repaired.exitCode, 0) assert.deepEqual(await fingerprint(), fresh) const repairedCitations = await sql< Array<{ chunk_id: string; source_link: string; embedding: string }> >`SELECT chunk_id, source_link, embedding::text FROM docs_embeddings WHERE source_document = ${page.sourcePath} ORDER BY chunk_id` assert.deepEqual(repairedCitations, citations) assert.equal((await index('check', 'citation-fresh', ['/introduction'])).exitCode, 0) } ) await check('Concurrent publishers fail before modifying any rows', async () => { await sql`SELECT pg_advisory_lock(724130561)` try { const result = await index('publish', 'concurrent-publish', ['/introduction']) assert.equal(result.exitCode, 1) assert(result.report.includes('Another documentation index publisher or check is active')) assert.deepEqual(await fingerprint(), fresh) } finally { await sql`SELECT pg_advisory_unlock(724130561)` } }) await check('Freshness rejects modified chunk bytes and embedding model identities', async () => { const [row] = await sql< Array<{ chunk_id: string; chunk_text: string }> >`SELECT chunk_id, chunk_text FROM docs_embeddings WHERE source_document = ${page.sourcePath} LIMIT 1` assert(row) await sql`UPDATE docs_embeddings SET chunk_text = 'corrupt stored chunk' WHERE chunk_id = ${row.chunk_id}` assert.equal((await index('check', 'corrupt-text', ['/introduction'])).exitCode, 1) const failure = await index('publish', 'corrupt-no-key', ['/introduction']) assert.equal(failure.exitCode, 1) assert(failure.report.includes('OPENAI_API_KEY is required')) assert.deepEqual(await fingerprint(), fresh) await sql`UPDATE docs_embeddings SET chunk_text = ${row.chunk_text}, embedding_model = 'wrong-model' WHERE chunk_id = ${row.chunk_id}` assert.equal((await index('check', 'wrong-model', ['/introduction'])).exitCode, 1) await sql`UPDATE docs_embeddings SET embedding_model = 'text-embedding-3-small' WHERE chunk_id = ${row.chunk_id}` assert.equal((await index('check', 'restored-chunk', ['/introduction'])).exitCode, 0) }) await sql` INSERT INTO docs_embeddings (chunk_id, chunk_text, source_document, source_link, header_text, header_level, token_count, embedding, metadata) VALUES (${generateId()}, 'retired fixture', 'retired-fixture.mdx', 'https://docs.sim.ai/retired-fixture', 'Retired', 1, 1, ${vector}::vector, '{}') ` await check( 'Freshness checks reject retired pages and publication retires only those rows', async () => { const check = await index('check', 'retired-check') assert.equal(check.exitCode, 1) assert(check.report.includes('removed documents remain indexed')) assert.equal((await index('publish', 'retired-publish')).exitCode, 0) assert.deepEqual(await fingerprint(), fresh) } ) const first = pages[0] assert(first) const source = [...pageSearchChunks(first)][0]?.sourceDocument assert(source) await sql`UPDATE docs_embeddings SET metadata = '{"contentHash":"stale"}' WHERE source_document = ${source}` await check( 'A missing embedding key fails publication without deleting old document data', async () => { const before = await fingerprint() const result = await index('publish', 'failed-publish') assert.equal(result.exitCode, 1) assert(result.report.includes('OPENAI_API_KEY is required')) assert.deepEqual(await fingerprint(), before) assert.equal((await index('check', 'stale-check')).exitCode, 1) } ) const responseValues = new Map() const requestSchema = z.object({ input: z.array(z.string().min(1).max(8000)).min(1).max(16), model: z.literal('text-embedding-3-small'), dimensions: z.literal(1536), encoding_format: z.literal('float'), }) provider = serve({ hostname: '127.0.0.1', port: 0, async fetch(request) { assert.equal(new URL(request.url).pathname, '/v1/embeddings') assert.equal(request.method, 'POST') assert.equal(request.headers.get('authorization'), 'Bearer isolated-fixture') const body = requestSchema.parse(await request.json()) scenarioRequests += 1 assert(providerRequests.length < 16, 'Provider fixture request cap exceeded') const status = providerScenario === '401' ? 401 : providerScenario === 'long-retry' ? 429 : providerScenario === 'batch-failure' && scenarioRequests > 1 ? 503 : (providerScenario === '429' || providerScenario === '503') && scenarioRequests === 1 ? Number(providerScenario) : 200 providerRequests.push({ scenario: providerScenario, attempt: scenarioRequests, status, inputs: body.input.length, }) if (status !== 200) return Response.json( { error: { message: 'Isolated fixture failure' } }, { status, headers: { 'Retry-After': providerScenario === 'long-retry' ? '120' : '0' } } ) const data = body.input .map((input, index) => { responseValues.set(input, index + 1) return { index: providerScenario === 'indices' ? body.input.length : index, embedding: [ index + 1, ...Array.from({ length: providerScenario === 'dimensions' ? 1534 : 1535 }, () => 0), ], } }) .reverse() return Response.json({ data, usage: { total_tokens: body.input.length } }) }, }) await writeFile( preloadPath, ` const endpoint = ${JSON.stringify(new URL('/v1/embeddings', provider.url).href)}; const transport = globalThis.fetch; globalThis.fetch = Object.assign(async (input, init) => { const request = new Request(input, init); if (request.url !== 'https://api.openai.com/v1/embeddings') throw new Error('Unexpected provider URL in isolated test'); return transport(new Request(endpoint, request)); }, transport); ` ) const introChunks = [...pageSearchChunks(page)] assert( introChunks.length > 0 && introChunks.length <= 16, 'Introduction must fit one bounded fixture batch' ) await check( 'First publication writes provider vectors in input order and an unchanged repeat makes no requests', async () => { await sql`DELETE FROM docs_embeddings WHERE source_document = ${page.sourcePath}` scenario('success') assert.equal((await index('publish', 'provider-first', ['/introduction'], true)).exitCode, 0) assert.equal(scenarioRequests, 1) const rows = await sql< Array<{ chunk_text: string header_text: string source_link: string embedding: string metadata: { title: string } }> >` SELECT chunk_text, header_text, source_link, embedding::text, metadata FROM docs_embeddings WHERE source_document = ${page.sourcePath} ` assert.equal(rows.length, providerRequests.at(-1)?.inputs) for (const row of rows) { const input = `${row.metadata.title}\n${row.header_text}\n${row.chunk_text}` const values = z.array(z.number()).parse(JSON.parse(row.embedding)) assert.equal(values.length, 1536) assert.equal(values[0], responseValues.get(input)) assert.equal(new URL(row.source_link).pathname, '/introduction') } const before = await documentState(page.sourcePath) assert.equal( (await index('publish', 'provider-unchanged', ['/introduction'], true)).exitCode, 0 ) assert.equal(scenarioRequests, 1) assert.deepEqual(await documentState(page.sourcePath), before) } ) await check( 'Changed stored chunk bytes are replaced while retaining other row identities', async () => { const rows = await sql< Array<{ chunk_id: string }> >`SELECT chunk_id FROM docs_embeddings WHERE source_document = ${page.sourcePath} ORDER BY chunk_id` const changed = rows[0] assert(changed) await sql`UPDATE docs_embeddings SET chunk_text = 'stale provider fixture' WHERE chunk_id = ${changed.chunk_id}` scenario('success') assert.equal( (await index('publish', 'provider-changed', ['/introduction'], true)).exitCode, 0 ) assert.equal(scenarioRequests, 1) assert.equal(providerRequests.at(-1)?.inputs, 1) const after = await sql< Array<{ chunk_id: string }> >`SELECT chunk_id FROM docs_embeddings WHERE source_document = ${page.sourcePath}` assert.equal(after.length, rows.length) assert(!after.some((row) => row.chunk_id === changed.chunk_id)) for (const retained of rows.slice(1)) assert(after.some((row) => row.chunk_id === retained.chunk_id)) } ) for (const status of ['429', '503'] as const) { await check( `Provider HTTP ${status} retries once and publishes the successful response`, async () => { await sql`UPDATE docs_embeddings SET chunk_text = 'stale provider fixture' WHERE chunk_id = (SELECT chunk_id FROM docs_embeddings WHERE source_document = ${page.sourcePath} LIMIT 1)` scenario(status) assert.equal( (await index('publish', `provider-retry-${status}`, ['/introduction'], true)).exitCode, 0 ) assert.equal(scenarioRequests, 2) assert.equal(providerRequests.at(-2)?.status, Number(status)) assert.equal(providerRequests.at(-1)?.status, 200) assert.equal( (await index('check', `provider-fresh-${status}`, ['/introduction'])).exitCode, 0 ) } ) } for (const failure of ['401', 'dimensions', 'indices', 'long-retry'] as const) { await check( `Provider ${failure} fails without retry or modifying the previous document`, async () => { await sql`UPDATE docs_embeddings SET chunk_text = 'stale provider fixture' WHERE chunk_id = (SELECT chunk_id FROM docs_embeddings WHERE source_document = ${page.sourcePath} LIMIT 1)` const before = await documentState(page.sourcePath) scenario(failure) const result = await index( 'publish', `provider-failure-${failure}`, ['/introduction'], true ) assert.equal(result.exitCode, 1) assert.equal(scenarioRequests, 1) assert.deepEqual(await documentState(page.sourcePath), before) } ) } await check( 'A successful first batch followed by persistent failure leaves the entire previous document intact', async () => { const multiBatch = pages.find((candidate) => { const count = [...pageSearchChunks(candidate)].length return count > 16 && count <= 32 }) assert(multiBatch, 'A bounded two-batch document is required') const source = [...pageSearchChunks(multiBatch)][0]?.sourceDocument assert(source) await sql`UPDATE docs_embeddings SET chunk_text = 'stale batch fixture' WHERE source_document = ${source}` const before = await documentState(source) scenario('batch-failure') const result = await index('publish', 'provider-partial-failure', [multiBatch.url], true) assert.equal(result.exitCode, 1) assert(scenarioRequests >= 2 && scenarioRequests <= 4) const attempts = providerRequests.filter((row) => row.scenario === 'batch-failure') assert.equal(attempts[0]?.status, 200) assert.equal(attempts[0]?.inputs, 16) assert(attempts.slice(1).every((row) => row.status === 503)) assert.deepEqual(await documentState(source), before) } ) } try { await check( 'Compiled publication chunks preserve code, tables, context and bounded payloads', verifyChunks ) if (!chunksOnly) await verifyDatabase() } catch (error) { if (!results.some((result) => result.status === 'failed')) { results.push({ name: 'Fixture lifecycle', status: 'failed', durationMs: 0, error: getErrorMessage(error), }) } } finally { searchProxy?.resume() await provider?.stop(true) await rm(preloadPath, { force: true }) if (created) await sql`DROP TABLE docs_embeddings` await Promise.all([sql.end({ timeout: 5 }), searchDatabase().end({ timeout: 5 })]) await searchProxy?.close() await writeFile( reportPath, `${JSON.stringify({ scope: chunksOnly ? 'compiled-content' : 'postgres-publication', results, provider: { mode: 'loopback-http-fixture', maxRequests: 16, maxInputsPerRequest: 16, requests: providerRequests } }, null, 2)}\n` ) } logger.info( chunksOnly ? 'Verified compiled search chunks' : 'Verified isolated PostgreSQL search boundary', { checks: results.length, reportPath } ) if (results.some((result) => result.status === 'failed')) process.exitCode = 1