Files
Nick Vigilante bcedc5d47e fix(scripts/docsgenfiltercheck): accept CRLF line endings in banner check (#30476)
`bodyOpensWithContentBanner` split pages on `\n` and compared lines
exactly.
On a Windows checkout with CRLF endings, the first line of
`docs/reference/cli/index.md` is `---\r`, so the front matter was never
recognized and `TestBodyOpensWithContentBanner` failed deterministically
on `test-go-pg (windows-2022)`.

Normalize CRLF to LF before parsing, and add CRLF table cases so the bug
reproduces on Linux too.

Fixes CLI-55.
Failing run: https://github.com/coder/coder/actions/runs/37570448753

## Testing

- The new CRLF cases failed before the fix and pass after it.
- `go test ./scripts/docsgenfiltercheck`, `GOOS=windows go vet`,
`golangci-lint`, and the full pre-commit hook pass.

<details>
<summary>CI flake note</summary>

The first `fmt` run failed while downloading Go modules
(`proxy.golang.org` returned `stream error: ... INTERNAL_ERROR` for
`testcontainers-go@v0.40.0`), which also failed `required`. This is a
network error unrelated to the change. I re-ran the failed jobs.

</details>

---

_This PR was generated by Coder Agents on behalf of @nickvigilante._
2026-10-08 12:24:42 -04:00

242 lines
7.7 KiB
Go

// Command docsgenfiltercheck fails when the docs-gen paths filter in the CI
// workflow drifts from the files make gen writes.
//
// CI skips the gen job for pull requests whose changed files all match the
// docs filter and none match the docs-gen filter. That is only safe while
// docs-gen matches every generated file that docs matches. The generated files
// are the arguments plus every tracked Markdown file the docs filter matches
// whose body opens with docgenenv.GeneratedContentBanner, which covers pages
// GEN_FILES doesn't list, such as the nested CLI and REST API reference pages.
// The check fails when a generated file matches docs but not docs-gen, and
// when a docs-gen pattern matches no tracked file, so stale entries don't
// accumulate.
//
// Usage:
//
// docsgenfiltercheck [-workflow path] generated-file ...
package main
import (
"errors"
"flag"
"fmt"
"io/fs"
"os"
"os/exec"
"regexp"
"slices"
"strings"
"golang.org/x/xerrors"
"gopkg.in/yaml.v3"
"github.com/coder/coder/v2/scripts/docgenenv"
)
// bodyOpensWithContentBanner reports whether the first non-blank line after
// the front matter of a Markdown page is docgenenv.GeneratedContentBanner,
// where the doc generators write it. Pages that only quote the banner
// elsewhere, such as in a code block, don't count. CRLF line endings, as in a
// Windows checkout, are treated as LF.
func bodyOpensWithContentBanner(page []byte) bool {
lines := strings.Split(strings.ReplaceAll(string(page), "\r\n", "\n"), "\n")
i := 0
if len(lines) > 0 && lines[0] == "---" {
end := slices.Index(lines[1:], "---")
if end < 0 {
return false
}
i = end + 2
}
for ; i < len(lines); i++ {
if strings.TrimSpace(lines[i]) != "" {
return lines[i] == docgenenv.GeneratedContentBanner
}
}
return false
}
func main() {
workflowPath := flag.String("workflow", ".github/workflows/ci.yaml", "Path to the workflow that defines the changes filter.")
flag.Parse()
if err := run(*workflowPath, flag.Args()); err != nil {
_, _ = fmt.Fprintf(os.Stderr, "docsgenfiltercheck: %v\n", err)
os.Exit(1)
}
}
func run(workflowPath string, generated []string) error {
if len(generated) == 0 {
return xerrors.New("no generated files given")
}
workflow, err := os.ReadFile(workflowPath)
if err != nil {
return xerrors.Errorf("read workflow: %w", err)
}
filters, err := parseFilters(workflow)
if err != nil {
return xerrors.Errorf("parse %s: %w", workflowPath, err)
}
lsFiles := exec.Command("git", "ls-files", "-z")
// Surface git's own message, such as the dubious-ownership hint.
lsFiles.Stderr = os.Stderr
out, err := lsFiles.Output()
if err != nil {
return xerrors.Errorf("git ls-files: %w", err)
}
tracked := strings.Split(strings.TrimSuffix(string(out), "\x00"), "\x00")
problems, err := check(filters, generated, tracked, readIfExists)
if err != nil {
return err
}
if len(problems) > 0 {
for _, p := range problems {
_, _ = fmt.Fprintf(os.Stderr, "%s: %s\n", workflowPath, p)
}
return xerrors.Errorf("%d problem(s) with the docs-gen filter", len(problems))
}
return nil
}
// readIfExists reads path, returning nil for a tracked file deleted from the
// working tree.
func readIfExists(path string) ([]byte, error) {
b, err := os.ReadFile(path)
if errors.Is(err, fs.ErrNotExist) {
return nil, nil
}
return b, err
}
// parseFilters returns the dorny/paths-filter filters of the step with id
// "filter" in the "changes" job.
func parseFilters(workflow []byte) (map[string][]string, error) {
var wf struct {
Jobs map[string]struct {
Steps []struct {
ID string `yaml:"id"`
With map[string]any `yaml:"with"`
} `yaml:"steps"`
} `yaml:"jobs"`
}
if err := yaml.Unmarshal(workflow, &wf); err != nil {
return nil, xerrors.Errorf("unmarshal workflow: %w", err)
}
var raw string
for _, step := range wf.Jobs["changes"].Steps {
if step.ID != "filter" {
continue
}
s, ok := step.With["filters"].(string)
if !ok {
return nil, xerrors.New(`changes job step "filter" has no string "filters" input`)
}
raw = s
}
if raw == "" {
return nil, xerrors.New(`changes job has no step with id "filter"`)
}
var entries map[string][]any
if err := yaml.Unmarshal([]byte(raw), &entries); err != nil {
return nil, xerrors.Errorf("unmarshal filters: %w", err)
}
filters := make(map[string][]string, len(entries))
for name, rules := range entries {
for _, rule := range rules {
// Status-qualified rules such as "added|modified: glob" decode
// as maps. fmt.Sprint renders them as "map[added|modified:glob]",
// and globRegexp rejects that text because of the brackets, so a
// status-qualified rule in docs or docs-gen fails the lint.
filters[name] = append(filters[name], fmt.Sprint(rule))
}
}
return filters, nil
}
// check returns the problems with the docs-gen filter. Tracked docs pages
// whose body opens with the generated-content banner count as generated
// alongside the given files.
func check(filters map[string][]string, generated, tracked []string, read func(string) ([]byte, error)) ([]string, error) {
docs, err := compileFilter(filters, "docs")
if err != nil {
return nil, err
}
docsGen, err := compileFilter(filters, "docs-gen")
if err != nil {
return nil, err
}
for _, f := range tracked {
if !strings.HasSuffix(f, ".md") || !matchesAny(docs, f) || slices.Contains(generated, f) {
continue
}
b, err := read(f)
if err != nil {
return nil, xerrors.Errorf("read %s: %w", f, err)
}
if bodyOpensWithContentBanner(b) {
generated = append(generated, f)
}
}
var problems []string
for _, f := range generated {
if matchesAny(docs, f) && !matchesAny(docsGen, f) {
problems = append(problems, fmt.Sprintf("make gen writes %s, which the docs filter matches; add a docs-gen pattern for it, or docs-only PRs that edit it skip gen", f))
}
}
for i, re := range docsGen {
if !slices.ContainsFunc(tracked, re.MatchString) {
problems = append(problems, fmt.Sprintf("docs-gen pattern %q matches no tracked file; remove it or fix the path", filters["docs-gen"][i]))
}
}
return problems, nil
}
func compileFilter(filters map[string][]string, name string) ([]*regexp.Regexp, error) {
patterns, ok := filters[name]
if !ok || len(patterns) == 0 {
return nil, xerrors.Errorf("filter %q is missing or empty", name)
}
res := make([]*regexp.Regexp, 0, len(patterns))
for _, p := range patterns {
re, err := globRegexp(p)
if err != nil {
return nil, xerrors.Errorf("filter %q: %w", name, err)
}
res = append(res, re)
}
return res, nil
}
func matchesAny(res []*regexp.Regexp, path string) bool {
return slices.ContainsFunc(res, func(re *regexp.Regexp) bool { return re.MatchString(path) })
}
// globRegexp converts the glob forms the docs and docs-gen filters use: a
// literal path, "*" within one path segment, and a literal directory followed
// by "/**", which matches the directory and everything under it, as picomatch
// does. Any other syntax is rejected, so a pattern this check might read
// differently from dorny/paths-filter fails the lint instead of matching
// wrongly.
func globRegexp(glob string) (*regexp.Regexp, error) {
base, recursive := strings.CutSuffix(glob, "/**")
unsupported := base == "" || strings.HasPrefix(base, "!") || strings.Contains(base, "**") || strings.ContainsAny(base, `?[]{}()+@\`)
// picomatch does not match "a/b" against "a/*/**", but the optional
// suffix below would, so keep the recursive form to literal directories.
if recursive && strings.Contains(base, "*") {
unsupported = true
}
if unsupported {
return nil, xerrors.Errorf("unsupported pattern %q: use a literal path, * within one segment, or a literal directory followed by /**", glob)
}
expr := strings.ReplaceAll(regexp.QuoteMeta(base), `\*`, "[^/]*")
if recursive {
expr += "(?:/.*)?"
}
return regexp.Compile("^" + expr + "$")
}