Files
sim/apps/docs/scripts/test-gitlab-export-e2e.py
Waleed 708e4b2b22 feat(search): import large permission snapshots in bounded parts (#10191)
* feat(search): import large permission snapshots in bounded parts

* fix(search): validate permission upload manifests and file inputs

* fix(search): reject exhausted permission import revisions
2026-11-18 16:20:38 -08:00

375 lines
23 KiB
Python

#!/usr/bin/env python3
"""Exercise the downloadable exporter over HTTPS, in a separate process."""
import csv
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import runpy
import ssl
import subprocess
import sys
import tempfile
import threading
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import parse_qs, urlsplit
SCRIPT = Path(__file__).resolve().parents[1] / "public/static/examples/gitlab/export-permissions.py"
TOKEN = "fixture-token-must-not-appear-in-output"
SIM_ROOT = Path(__file__).resolve().parents[2] / "sim"
def member(user_id, **overrides):
return {"id": user_id, "state": "active", "access_level": 30,
"expires_at": None, **overrides}
class GitLabFixture(BaseHTTPRequestHandler):
calls = []
def log_message(self, *_args):
pass
def do_GET(self):
parsed = urlsplit(self.path)
scenario, endpoint = parsed.path.lstrip("/").split("/api/v4/", 1)
self.calls.append((scenario, endpoint, parse_qs(parsed.query), time.monotonic()))
if self.headers.get("PRIVATE-TOKEN") != TOKEN:
self.reply(401, {"message": "Missing token"})
return
if scenario == "denied":
self.reply(403, {"message": TOKEN})
return
if scenario.startswith("group"):
if endpoint in ("groups/engineering", "groups/7"):
self.reply(200, {"id": 7, "full_path": "engineering"})
return
if endpoint == "groups/7/projects":
query = parse_qs(parsed.query)
assert query.get("with_shared") == ["false"]
assert query.get("include_subgroups") == ["true" if scenario == "group-subgroups" else "false"]
if scenario in ("group-large", "group-too-large"):
if scenario == "group-too-large":
self.reply(200, [], {"X-Total": "50001"})
else:
page = int(query.get("page", ["1"])[0])
self.reply(200, [{"id": index, "path_with_namespace": f"engineering/project-{index}"}
for index in range(10000 + (page - 1) * 100, min(11200, 10000 + page * 100))])
return
projects = [{"id": 91, "path_with_namespace": "engineering/platform"}]
if query.get("page", ["1"]) == ["1"]:
self.reply(200, projects, {"X-Next-Page": "2", "X-Total": "2"})
elif scenario == "group-failure":
self.reply(503, {})
else:
path = "outside/shared" if scenario == "group-outside" else (
"engineering/sub/backend" if scenario == "group-subgroups" else "engineering/backend")
self.reply(200, [{"id": 92, "path_with_namespace": path}], {"X-Next-Page": "", "X-Total": "2"})
return
if endpoint == "projects/92":
self.reply(200, {"id": 92, "path_with_namespace": (
"engineering/sub/backend" if scenario == "group-subgroups" else "engineering/backend")})
return
if endpoint in ("projects/91/members/all", "projects/92/members/all"):
if scenario == "group-member-failure" and endpoint.startswith("projects/92/"):
self.reply(403, {})
return
members = [member(1)] if endpoint.startswith("projects/91/") else [member(1), member(2)]
if scenario == "group-duplicate-email" and endpoint.startswith("projects/92/"):
members = [member(2)]
if scenario == "group-empty-project" and endpoint.startswith("projects/92/"):
members = []
self.reply(200, members, {"X-Next-Page": "", "X-Total": str(len(members))})
return
if scenario == "group-large" and endpoint.startswith("projects/"):
project_id = int(endpoint.split("/")[1])
if endpoint.endswith("/members/all"):
self.reply(200, [member(1)] if project_id == 11199 else [])
else:
self.reply(200, {"id": project_id, "path_with_namespace": f"engineering/project-{project_id}"})
return
if endpoint in ("projects/engineering%2Fplatform", "projects/91"):
if scenario == "redirect":
self.reply(302, {}, {"Location": "/redirect/api/v4/trap"})
else:
self.reply(200, {"id": 91, "path_with_namespace": "engineering/platform",
"repository_access_level": "disabled" if scenario == "issues-only" else "enabled",
"issues_access_level": "enabled", "wiki_access_level": "enabled"})
elif endpoint == "projects/91/members/all":
page = parse_qs(parsed.query).get("page", ["1"])[0]
total = "11" if scenario in ("truncated-membership", "changing-total") else "10"
if scenario == "changing-total" and page == "2":
total = "10"
headers = {} if scenario == "missing-pagination" else {"X-Total": total}
if scenario == "page-failure" and page == "2":
self.reply(500, {"message": TOKEN})
elif scenario == "pagination-loop":
self.reply(200, [member(1)], {"X-Next-Page": "1"})
elif scenario == "missing-pagination" and page == "1":
self.reply(200, [member(i, access_level=10) for i in range(100, 200)])
elif scenario == "empty":
self.reply(200, [member(3, access_level=10)])
elif page == "1":
self.reply(200, [member(2), member(1)], {**headers, "X-Next-Page": "2"})
else:
self.reply(200, [member(3, access_level=10), member(4, state="blocked"),
member(5), member(6, expires_at="2000-01-01"),
member(7), member(8, access_level=15), member(9),
member(10, expires_at=datetime.now(timezone.utc).date().isoformat())],
{**headers, "X-Next-Page": ""})
elif endpoint.startswith("users/"):
user_id = int(endpoint.split("/")[1])
email = {1: "ALICE@example.com", 2: "bob@example.com", 7: "carol@example.com"}.get(
user_id, f"service-{user_id}@example.com")
if scenario in ("duplicate-email", "group-duplicate-email") and user_id == 2:
email = "alice@example.com"
profile = {"id": user_id, "state": "active", "locked": user_id == 9,
"bot": user_id == 5, "email": email, "confirmed_at": "2025-01-01T00:00:00Z"}
if user_id == 2:
if scenario == "unconfirmed-email":
profile["confirmed_at"] = None
elif scenario == "missing-confirmation":
profile.pop("confirmed_at")
elif scenario == "invalid-confirmation":
profile["confirmed_at"] = "not-a-timestamp"
if scenario == "hidden-email":
profile.pop("email")
profile["public_email"] = f"public-alias-{user_id}@example.com"
if scenario == "wrong-identity":
profile["id"] = 999
self.reply(200, profile)
else:
self.reply(404, {"message": "Unexpected endpoint"})
def reply(self, status, body, headers=None):
data = json.dumps(body).encode()
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(data)))
for key, value in (headers or {}).items():
self.send_header(key, value)
self.end_headers()
self.wfile.write(data)
def run():
report_path = Path(os.environ["GITLAB_EXPORT_REPORT_PATH"])
report = {"boundary": "CLI subprocess against a local HTTPS GitLab API fixture", "checks": []}
try:
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
cert, key = root / "cert.pem", root / "key.pem"
subprocess.run([
"openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "1",
"-keyout", str(key), "-out", str(cert), "-subj", "/CN=localhost",
"-addext", "subjectAltName=IP:127.0.0.1",
], check=True, capture_output=True)
server = ThreadingHTTPServer(("127.0.0.1", 0), GitLabFixture)
tls = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
tls.load_cert_chain(cert, key)
server.socket = tls.wrap_socket(server.socket, server_side=True)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
for scenario, expected_error in [
("success", None), ("numeric-project", None), ("missing-pagination", None),
("issues-only", None), ("paced", None),
("unconfirmed-email", None), ("missing-confirmation", None),
("invalid-confirmation", None),
("truncated-membership", "total"), ("changing-total", "changed"),
("hidden-email", "email"), ("duplicate-email", "email"),
("wrong-identity", "identity"), ("denied", "403"),
("page-failure", "500"), ("pagination-loop", "pagination"),
("redirect", "302"), ("empty", "readers"),
("existing-output", "exists"),
]:
output = root / scenario
if scenario == "existing-output":
output.mkdir()
(output / "users.csv").write_text("keep the previous export")
command = [sys.executable, str(SCRIPT), "--gitlab-url",
f"https://127.0.0.1:{server.server_port}/{scenario}",
"--project", "91" if scenario == "numeric-project" else "engineering/platform",
"--output-dir", str(output),
"--requests-per-minute", "120" if scenario == "paced" else "600000"]
result = subprocess.run(command, capture_output=True, text=True, timeout=20,
env={**os.environ, "GITLAB_EXPORT_TOKEN": TOKEN,
"SSL_CERT_FILE": str(cert)})
assert TOKEN not in result.stdout + result.stderr, "Token leaked in output"
if expected_error:
assert result.returncode != 0, f"{scenario}: export unexpectedly succeeded"
assert expected_error in result.stderr.lower(), (scenario, result.stderr)
if scenario == "existing-output":
assert (output / "users.csv").read_text() == "keep the previous export"
else:
assert not output.exists(), f"{scenario}: published a partial export"
else:
assert result.returncode == 0, (scenario, result.stderr)
with (output / "users.csv").open() as handle:
users = list(csv.reader(handle))
with (output / "project-permissions.csv").open() as handle:
grants = list(csv.reader(handle))
expected = [["7", "carol@example.com"]] if scenario == "missing-pagination" else [
["1", "alice@example.com"], ["2", "bob@example.com"],
["7", "carol@example.com"]]
if scenario in ("unconfirmed-email", "missing-confirmation", "invalid-confirmation"):
expected = [["1", "alice@example.com"], ["7", "carol@example.com"]]
assert users == [["user_id", "email"], *expected], users
assert grants == [["project_path", "user_id"], *[
["engineering/platform", row[0]] for row in expected]], grants
parsed = subprocess.run([
"bun", "-e", """
import { parseGitLabCsv } from './connectors/gitlab/permission-config/parser'
const directory = process.argv[1]
const users = parseGitLabCsv({filename: 'users.csv', content: await Bun.file(`${directory}/users.csv`).text()}, 'userMapping')
const grants = parseGitLabCsv({filename: 'project-permissions.csv', content: await Bun.file(`${directory}/project-permissions.csv`).text()}, 'projectPermissions')
process.stdout.write(JSON.stringify({
users, grants,
}))
""", str(output),
], cwd=SIM_ROOT, capture_output=True, text=True, timeout=20)
assert parsed.returncode == 0, parsed.stderr
assert json.loads(parsed.stdout) == {
"users": expected,
"grants": [["engineering/platform", row[0]] for row in expected],
}, parsed.stdout
if scenario == "paced":
arrivals = [arrived for case, _, _, arrived in GitLabFixture.calls if case == scenario]
# The first request's transit time can consume one pacing interval.
minimum_span = (len(arrivals) - 2) * 0.5
assert arrivals[-1] - arrivals[0] >= minimum_span, "Requests exceeded the configured rate"
report["pacing"] = {"requests": len(arrivals), "requestsPerMinute": 120,
"observedSeconds": arrivals[-1] - arrivals[0],
"minimumSeconds": minimum_span}
report["checks"].append({"name": scenario, "passed": True})
for scenario in ("group", "group-subgroups", "group-empty-project", "group-outside", "group-failure", "group-member-failure", "group-duplicate-email"):
output = root / scenario
command = [sys.executable, str(SCRIPT), "--gitlab-url",
f"https://127.0.0.1:{server.server_port}/{scenario}", "--group", "engineering",
"--output-dir", str(output), "--requests-per-minute", "600000"]
if scenario == "group-subgroups":
command.append("--include-subgroups")
result = subprocess.run(command, capture_output=True, text=True, timeout=20,
env={**os.environ, "GITLAB_EXPORT_TOKEN": TOKEN, "SSL_CERT_FILE": str(cert)})
assert TOKEN not in result.stdout + result.stderr
if scenario in ("group-outside", "group-failure", "group-member-failure", "group-duplicate-email"):
assert result.returncode != 0 and not output.exists(), (scenario, result.stderr)
error = {"group-outside": "outside", "group-failure": "503", "group-member-failure": "403", "group-duplicate-email": "email"}[scenario]
assert error in result.stderr.lower(), (scenario, result.stderr)
else:
assert result.returncode == 0, (scenario, result.stderr)
with (output / "users.csv").open() as handle:
users = list(csv.reader(handle))
with (output / "project-permissions.csv").open() as handle:
grants = list(csv.reader(handle))
expected_users = [["user_id", "email"], ["1", "alice@example.com"]]
expected_grants = [["project_path", "user_id"], ["engineering/platform", "1"]]
if scenario != "group-empty-project":
expected_users.append(["2", "bob@example.com"])
path = "engineering/sub/backend" if scenario == "group-subgroups" else "engineering/backend"
expected_grants.extend([[path, "1"], [path, "2"]])
assert users == expected_users, users
assert sorted(grants[1:]) == sorted(expected_grants[1:]), grants
profiles = [endpoint for case, endpoint, _, _ in GitLabFixture.calls
if case == scenario and endpoint.startswith("users/")]
assert len(profiles) == len(set(profiles)), "Fetched a shared user profile more than once"
report["checks"].append({"name": scenario, "passed": True})
for index, group in enumerate((".", "..", "%2e", "%2e%2e", ".%2e", "%2e.")):
output = root / f"invalid-group-{index}"
calls = len(GitLabFixture.calls)
result = subprocess.run([
sys.executable, str(SCRIPT), "--gitlab-url",
f"https://127.0.0.1:{server.server_port}/group", "--group", group,
"--output-dir", str(output),
], capture_output=True, text=True, timeout=20,
env={**os.environ, "GITLAB_EXPORT_TOKEN": TOKEN, "SSL_CERT_FILE": str(cert)})
assert result.returncode != 0 and not output.exists()
assert TOKEN not in result.stdout + result.stderr
assert len(GitLabFixture.calls) == calls, "Sent a token for an invalid group identifier"
report["checks"].append({"name": "invalid-group-identifiers", "passed": True})
for scenario in ("group-large", "group-too-large"):
output = root / scenario
result = subprocess.run([
sys.executable, str(SCRIPT), "--gitlab-url",
f"https://127.0.0.1:{server.server_port}/{scenario}", "--group", "engineering",
"--output-dir", str(output), "--requests-per-minute", "600000",
], capture_output=True, text=True, timeout=60,
env={**os.environ, "GITLAB_EXPORT_TOKEN": TOKEN, "SSL_CERT_FILE": str(cert)})
if scenario == "group-too-large":
assert result.returncode != 0 and not output.exists(), result.stderr
assert "50,000-project limit" in result.stderr, result.stderr
else:
assert result.returncode == 0, result.stderr
with (output / "project-permissions.csv").open() as handle:
assert list(csv.reader(handle)) == [
["project_path", "user_id"], ["engineering/project-11199", "1"]]
with (output / "users.csv").open() as handle:
assert list(csv.reader(handle)) == [["user_id", "email"], ["1", "alice@example.com"]]
report["checks"].append({"name": scenario, "passed": True})
exporter = runpy.run_path(str(SCRIPT))
output = root / "large-csv"
exporter["write_export"](output,
((f"engineering/project-{index:06d}-permissions", 1) for index in range(600001)),
{1: "alice@example.com"})
assert (output / "project-permissions.csv").stat().st_size > 8 * 1024 * 1024
with (output / "project-permissions.csv").open() as handle:
rows = csv.reader(handle)
assert next(rows) == ["project_path", "user_id"]
assert sum(1 for _ in rows) == 600001
report["checks"].append({"name": "large-csv", "passed": True})
writer_globals = exporter["write_export"].__globals__
# Exercise overflow cleanup without creating a 1 GiB test artifact.
writer_globals["MAX_BYTES"] = 8 * 1024 * 1024
writer_globals["MAX_GRANTS"] = 500000
for scenario, grants in (
("csv-row-bound", (("a/b", 1) for _ in range(500001))),
("csv-byte-bound", (("a/" + "b" * 1000, 1) for _ in range(9000))),
("csv-partial-failure", None),
):
if scenario == "csv-partial-failure":
def interrupted_grants():
yield ("a/b", 1)
raise exporter["ExportError"]("Provider failed after the first grant")
grants = interrupted_grants()
output = root / scenario
try:
exporter["write_export"](output, grants, {1: "alice@example.com"})
raise AssertionError(f"{scenario}: export unexpectedly succeeded")
except exporter["ExportError"] as error:
expected = {"csv-row-bound": "500,000-row limit", "csv-byte-bound": "CSV byte limit",
"csv-partial-failure": "Provider failed after the first grant"}[scenario]
assert expected in str(error), str(error)
assert not output.exists(), "Published an incomplete CSV pair"
report["checks"].append({"name": scenario, "passed": True})
writer_globals["MAX_BYTES"] = 60
output = root / "combined-csv-byte-bound"
try:
exporter["write_export"](output, iter([("a/b", 1)]),
{1: "alice@example.com", 2: "bob@example.com"})
raise AssertionError("Published two individually small CSVs exceeding the combined limit")
except exporter["ExportError"] as error:
assert "CSV byte limit" in str(error), str(error)
assert not output.exists(), "Published an oversized CSV pair"
report["checks"].append({"name": "combined-csv-byte-bound", "passed": True})
assert not any(endpoint == "trap" for _, endpoint, _, _ in GitLabFixture.calls)
report["passed"] = True
finally:
server.shutdown()
server.server_close()
thread.join()
except Exception as error:
report["passed"] = False
report["error"] = str(error)
raise
finally:
report_path.parent.mkdir(parents=True, exist_ok=True)
report_path.write_text(json.dumps(report, indent=2) + "\n")
print(f"Passed {len(report['checks'])} exporter checks. Report: {report_path}")
if __name__ == "__main__":
run()