mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-20 13:19:34 +08:00
* test(storage): restore POST form metadata policy fixture * test(storage): follow the current PUT path in contract checks * fix(storage): verify archive transport EOF before parsing the final frame * test(rustfs): restore main verification after gateway migration Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * refactor(storage): retire the legacy S3 storage edge * fix(storage): bound the audited multipart copy future and trace LIST quorum causes * fix(deps): align bindgen and prettyplease on syn 3 for FIPS builds * fix(storage): expose the typed legacy cause on owned errors * test(storage): preserve stream lengths and request facts on retired seams --------- Co-authored-by: Hauser Zhi <housemecn@gmail.com> Co-authored-by: heihutu <heihutu@gmail.com> Co-authored-by: zhi22915 <qiuzgang@gmail.com>
346 lines
16 KiB
Bash
Executable File
346 lines
16 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# End-to-end probe for the guards that must survive the mechanical split of
|
|
# bucket_usecase.rs, multipart_usecase.rs and
|
|
# lifecycle_transition_api_test.rs (rustfs/backlog#2765).
|
|
#
|
|
# It copies the working tree and works out, for bucket_usecase.rs and multipart_usecase.rs, whether that source is still
|
|
# unsplit or already split:
|
|
#
|
|
# unsplit the split directory holds no production .rs file; the probe
|
|
# simulates the split (moves real code into the directory) and
|
|
# plants its violations in new split files;
|
|
# split the split directory holds production .rs files and the source's
|
|
# anchor lives there; the probe plants the same violations at the
|
|
# top of real split files.
|
|
#
|
|
# A source in neither state (parent file missing, anchor missing, duplicated,
|
|
# or on the wrong side) fails the probe with a message naming it. Either way
|
|
# the real guard scripts run in the copy: each planted line must be reported
|
|
# with its split-file path, and the files holding only real or ordinary code
|
|
# must not be reported. The real workspace is never modified.
|
|
#
|
|
# SPLIT_PROBE_SOURCE_ROOT points the copy at another checkout (read-only), to
|
|
# exercise a split branch's tree with that branch's guard scripts.
|
|
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
SOURCE_ROOT="${SPLIT_PROBE_SOURCE_ROOT:-$ROOT_DIR}"
|
|
PYTHON="${RUSTFS_PYTHON_BIN:-python3}"
|
|
FX="$(mktemp -d)"
|
|
trap 'rm -rf "$FX"' EXIT
|
|
|
|
failures=0
|
|
checked=0
|
|
|
|
fail() {
|
|
echo "FAIL: $1" >&2
|
|
failures=$((failures + 1))
|
|
}
|
|
|
|
# expect_reported <output-file> <path> <text> <description>
|
|
expect_reported() {
|
|
checked=$((checked + 1))
|
|
if ! grep -F -- "$2" "$1" | grep -qF -- "$3"; then
|
|
fail "$4: no report naming $2 with: $3"
|
|
fi
|
|
}
|
|
|
|
# expect_silent <output-file> <path> <description>
|
|
expect_silent() {
|
|
checked=$((checked + 1))
|
|
if grep -qF -- "$2" "$1"; then
|
|
fail "$3: $2 was reported: $(grep -F -- "$2" "$1" | head -3)"
|
|
fi
|
|
}
|
|
|
|
# Copy the tracked and untracked-but-not-ignored files of the source checkout.
|
|
(cd "$SOURCE_ROOT" && git ls-files -z --cached --others --exclude-standard) |
|
|
"$PYTHON" -c '
|
|
import os, shutil, sys
|
|
root, dest = sys.argv[1], sys.argv[2]
|
|
for rel in sys.stdin.buffer.read().decode().split("\0"):
|
|
if not rel:
|
|
continue
|
|
src = os.path.join(root, rel)
|
|
if not os.path.lexists(src):
|
|
continue
|
|
out = os.path.join(dest, rel)
|
|
os.makedirs(os.path.dirname(out), exist_ok=True)
|
|
shutil.copy2(src, out, follow_symlinks=False)
|
|
' "$SOURCE_ROOT" "$FX"
|
|
|
|
# Decide each source's state, move real code where the tree is unsplit, and
|
|
# plant one forbidden line per rule. Prints shell assignments for the checks.
|
|
PLAN="$FX/.split-probe-plan.sh"
|
|
"$PYTHON" - "$FX" >"$PLAN" <<'PY'
|
|
import shlex
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
fx = Path(sys.argv[1])
|
|
APP = "rustfs/src/app"
|
|
STORAGE = "rustfs/src/storage"
|
|
|
|
|
|
def die(message):
|
|
print(f"split probe: {message}", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
|
|
def is_test_only(rel):
|
|
parts = rel.split("/")
|
|
stem = parts[-1][:-3] if parts[-1].endswith(".rs") else parts[-1]
|
|
return ("tests" in parts[:-1] or stem in ("tests", "test_support")
|
|
or stem.endswith("_test") or stem.endswith("_tests"))
|
|
|
|
|
|
def production_files(split_dir):
|
|
root = fx / split_dir
|
|
if not root.is_dir():
|
|
return []
|
|
return sorted(p.relative_to(fx).as_posix() for p in root.rglob("*.rs")
|
|
if not is_test_only(p.relative_to(fx).as_posix()))
|
|
|
|
|
|
def defining(rel_paths, marker):
|
|
return [rel for rel in rel_paths if marker in (fx / rel).read_text()]
|
|
|
|
|
|
def state(name, parent, split_dir, anchor):
|
|
"""'unsplit' or 'split'; anything else stops the probe naming the source."""
|
|
if not (fx / parent).is_file():
|
|
die(f"{name}: {parent} is missing, so the tree is in neither the unsplit nor the split state")
|
|
split_files = production_files(split_dir)
|
|
total = (fx / parent).read_text().count(anchor) + sum(
|
|
(fx / rel).read_text().count(anchor) for rel in split_files)
|
|
if total != 1:
|
|
die(f"{name}: anchor {anchor!r} occurs {total} times across {parent} and {split_dir}/, expected once")
|
|
in_parent = anchor in (fx / parent).read_text()
|
|
if not split_files:
|
|
if not in_parent:
|
|
die(f"{name}: {split_dir}/ has no production file but {anchor!r} is not in {parent}")
|
|
return "unsplit", split_files
|
|
return "split", split_files
|
|
|
|
|
|
def prepend(rel, block):
|
|
path = fx / rel
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
old = path.read_text() if path.exists() else ""
|
|
path.write_text(block + old)
|
|
|
|
|
|
def pick(split_files, avoid, fallback):
|
|
for rel in split_files:
|
|
if rel not in avoid and not rel.endswith("/mod.rs"):
|
|
return rel
|
|
return fallback
|
|
|
|
|
|
def imports(rel):
|
|
return "".join(l + "\n" for l in (fx / rel).read_text().splitlines() if l.startswith("use crate::"))
|
|
|
|
|
|
LISTING_ANCHOR = " pub async fn execute_list_objects_v2("
|
|
bu_state, bu_files = state("bucket usecase", f"{APP}/bucket_usecase.rs", f"{APP}/bucket_usecase", LISTING_ANCHOR)
|
|
mu_state, mu_files = state("multipart usecase", f"{APP}/multipart_usecase.rs", f"{APP}/multipart_usecase",
|
|
"pub async fn execute_complete_multipart_upload(")
|
|
|
|
# Bucket usecase: the TRACE hot span either moves into listing.rs (unsplit) or
|
|
# already lives in a real split file.
|
|
if bu_state == "unsplit":
|
|
parent = fx / APP / "bucket_usecase.rs"
|
|
text = parent.read_text()
|
|
idx = text.index(LISTING_ANCHOR)
|
|
start = text.rindex("\n", 0, idx) + 1
|
|
while True:
|
|
prev = text.rindex("\n", 0, start - 1) + 1
|
|
if text[prev:start].lstrip().startswith(("#[", "///")):
|
|
start = prev
|
|
else:
|
|
break
|
|
end = text.index("\n }\n", idx) + len("\n }\n")
|
|
listing = f"{APP}/bucket_usecase/listing.rs"
|
|
prepend(listing, "impl DefaultBucketUsecase {\n" + text[start:end] + "}\n")
|
|
parent.write_text(text[:start] + text[end:])
|
|
else:
|
|
owners = defining(bu_files, LISTING_ANCHOR)
|
|
if len(owners) != 1:
|
|
die(f"bucket usecase: {APP}/bucket_usecase/ is split but {LISTING_ANCHOR.strip()!r} is not in one of its files")
|
|
listing = owners[0]
|
|
|
|
lt_imports = f"{APP}/lifecycle_transition_api_test/split_probe_imports.rs"
|
|
prepend(lt_imports, imports(f"{APP}/lifecycle_transition_api_test.rs"))
|
|
|
|
e_file = f"{STORAGE}/legacy_edge/bucket_configs.rs"
|
|
t_file = f"{STORAGE}/helpers_test.rs"
|
|
b_file = pick(bu_files, {listing}, f"{APP}/bucket_usecase/cors.rs") if bu_state == "split" \
|
|
else f"{APP}/bucket_usecase/cors.rs"
|
|
m_file = pick(mu_files, set(), f"{APP}/multipart_usecase/complete.rs") if mu_state == "split" \
|
|
else f"{APP}/multipart_usecase/complete.rs"
|
|
l_file = f"{APP}/lifecycle_transition_api_test/split_probe.rs"
|
|
|
|
PLANTS = {
|
|
e_file: ("use crate::app::context::resolve_region;\n"
|
|
"use crate::app::bucket_usecase::DefaultBucketUsecase;\n"
|
|
"use crate::storage::core_storage_compat::Probe;\n"
|
|
"use crate::startup_server::LayerProbe;\n"),
|
|
b_file: ("use crate::storage::*;\n"
|
|
"use crate::storage::access::ReqInfo;\n"
|
|
"use super::super::StorageError;\n"
|
|
"use super::super::ECStore;\n"
|
|
"fn forbidden() {\n"
|
|
" super::super::get_server_info();\n"
|
|
"}\n"),
|
|
m_file: ("use crate::storage::s3_api::multipart;\n"
|
|
"use super::super::{quota, versioning_sys};\n"
|
|
"use crate::app::usecase_storage_compat::Probe;\n"),
|
|
l_file: ("use crate::storage::request_context::RequestContext;\n"
|
|
"fn forbidden() {\n"
|
|
" super::super::tagging::probe();\n"
|
|
"}\n"),
|
|
t_file: ("use super::{BucketMetadata};\n"
|
|
"mod tests {\n"
|
|
" use super::super::{check_preconditions};\n"
|
|
"}\n"
|
|
"use crate::storage::core_storage_compat::TestProbe;\n"),
|
|
}
|
|
if len(PLANTS) != 5:
|
|
die("two planted violation sets would share one file")
|
|
|
|
ORDINARY = ("use super::{DefaultBucketUsecase, ObjectInfo, ObjectOptions};\n"
|
|
"use super::ECStore;\n"
|
|
"use super::StorageError;\n"
|
|
"fn ordinary() {\n"
|
|
" super::tagging::helper();\n"
|
|
" super::get_server_info();\n"
|
|
"}\n")
|
|
ordinary = [f"{APP}/bucket_usecase/split_probe_ordinary.rs", f"{APP}/multipart_usecase/split_probe_ordinary.rs",
|
|
f"{APP}/lifecycle_transition_api_test/split_probe_ordinary.rs"]
|
|
for rel in ordinary:
|
|
if (fx / rel).exists():
|
|
die(f"{rel} already exists; the probe will not overwrite a real file")
|
|
prepend(rel, ORDINARY)
|
|
for rel, block in PLANTS.items():
|
|
prepend(rel, block)
|
|
if not (fx / rel).read_text().startswith(block):
|
|
die(f"planting into {rel} did not take effect")
|
|
|
|
# Real split files that carry no planted line hold moved production code only:
|
|
# no rule may report them.
|
|
real = [rel for rel in bu_files + mu_files if rel not in PLANTS and rel != listing]
|
|
|
|
print(f"BU_STATE={bu_state} MU_STATE={mu_state}")
|
|
for var, value in [("E", e_file), ("B", b_file), ("M", m_file), ("L", l_file), ("T", t_file),
|
|
("LISTING", listing), ("LT_IMPORTS", lt_imports)]:
|
|
print(f"{var}={shlex.quote(value)}")
|
|
print("SILENT_FILES=(" + " ".join(shlex.quote(p) for p in ordinary + [listing, lt_imports]) + ")")
|
|
print("REAL_SPLIT_FILES=(" + " ".join(shlex.quote(p) for p in real) + ")")
|
|
PY
|
|
# shellcheck source=/dev/null
|
|
source "$PLAN"
|
|
echo "split probe: bucket usecase $BU_STATE, multipart usecase $MU_STATE"
|
|
|
|
# --- architecture rules ------------------------------------------------------
|
|
ARCH_OUT="$FX/.arch.out"
|
|
if (cd "$FX" && ./scripts/check_architecture_migration_rules.sh) >"$ARCH_OUT" 2>&1; then
|
|
fail "architecture rules passed over planted violations"
|
|
fi
|
|
expect_reported "$ARCH_OUT" "rustfs/src/storage/legacy_edge" "Retired S3 storage edge reintroduced" "retired edge"
|
|
expect_reported "$ARCH_OUT" "$B:1:" "instead of crate::storage::*" "storage wildcard (bucket split)"
|
|
expect_reported "$ARCH_OUT" "$B:2:" "consume storage helper APIs through" "storage helper import (bucket split)"
|
|
expect_reported "$ARCH_OUT" "$B:3:" "consume storage helper APIs through" "depth-aware super::super storage helper"
|
|
expect_reported "$ARCH_OUT" "$B:4:" "runtime and data-usage facades must stay behind" "depth-aware super::super runtime facade import"
|
|
expect_reported "$ARCH_OUT" "$B:6:" "runtime and data-usage facades must stay behind" "depth-aware super::super runtime facade path"
|
|
expect_reported "$ARCH_OUT" "$M:1:" "consume S3 API helpers through" "s3_api bypass (multipart split)"
|
|
expect_reported "$ARCH_OUT" "$M:2:" "consume storage helper APIs through" "depth-aware super::super owner import"
|
|
expect_reported "$ARCH_OUT" "$M:3:" "local compatibility consumers must use relative" "usecase_storage_compat (multipart split)"
|
|
expect_reported "$ARCH_OUT" "$L:1:" "completed storage helper consumers" "test storage helper (lifecycle split)"
|
|
expect_reported "$ARCH_OUT" "$L:3:" "consume storage helper APIs through" "depth-aware super::super owner path"
|
|
expect_reported "$ARCH_OUT" "$T:3:" "storage-owner tests must use storage_api test consumer" "helper import in an inline tests module"
|
|
expect_reported "$ARCH_OUT" "$T:5:" "local compatibility consumers must use relative" "helper test core_storage_compat"
|
|
for silent in "${SILENT_FILES[@]}"; do
|
|
expect_silent "$ARCH_OUT" "$silent:" "architecture rules over moved or ordinary code"
|
|
done
|
|
for silent in ${REAL_SPLIT_FILES[@]+"${REAL_SPLIT_FILES[@]}"}; do
|
|
expect_silent "$ARCH_OUT" "$silent:" "architecture rules over real split code"
|
|
done
|
|
|
|
# --- layer dependencies ------------------------------------------------------
|
|
# Protocol edges may call the app; app code cannot call either public client path.
|
|
mkdir -p "$FX/rustfs/src/protocols/client"
|
|
cat >"$FX/rustfs/src/protocols/client/layer_probe.rs" <<'PROTOCOL_EDGE'
|
|
use crate::app::RequestEnvelope;
|
|
PROTOCOL_EDGE
|
|
cat >"$FX/rustfs/src/protocols/client_helper.rs" <<'PROTOCOL_NEIGHBOR'
|
|
use crate::app::RequestEnvelope;
|
|
PROTOCOL_NEIGHBOR
|
|
cat >"$FX/rustfs/src/app/protocol_client_layer_probe.rs" <<'PROTOCOL_REVERSE'
|
|
use crate::protocols::client::ProtocolStorageClient;
|
|
use crate::protocols::ProtocolStorageClient;
|
|
PROTOCOL_REVERSE
|
|
LAYER_OUT="$FX/.layer.out"
|
|
if (cd "$FX" && ./scripts/check_layer_dependencies.sh) >"$LAYER_OUT" 2>&1; then
|
|
fail "layer guard passed over a planted reverse dependency"
|
|
fi
|
|
expect_reported "$LAYER_OUT" "dep|$E|interface->composition|" "crate::startup_server::LayerProbe" "legacy_edge keeps the interface layer"
|
|
expect_silent "$LAYER_OUT" "dep|rustfs/src/app/lifecycle_transition_api_test/" "moved lifecycle transition tests stay excluded"
|
|
expect_silent "$LAYER_OUT" "dep|$LISTING|" "moved bucket usecase code keeps the app layer"
|
|
expect_silent "$LAYER_OUT" "dep|rustfs/src/protocols/client/layer_probe.rs|" "protocol client edge may use app contracts"
|
|
expect_reported "$LAYER_OUT" "dep|rustfs/src/protocols/client_helper.rs|infra->app|" "crate::app::RequestEnvelope" "neighboring protocol helper remains infrastructure"
|
|
expect_reported "$LAYER_OUT" "dep|rustfs/src/app/protocol_client_layer_probe.rs|app->interface|" "crate::protocols::client::ProtocolStorageClient" "app cannot import the protocol client module"
|
|
expect_reported "$LAYER_OUT" "dep|rustfs/src/app/protocol_client_layer_probe.rs|app->interface|" "crate::protocols::ProtocolStorageClient" "app cannot import the protocol client re-export"
|
|
|
|
# --- logging hot span --------------------------------------------------------
|
|
LOG_OUT="$FX/.logging.out"
|
|
checked=$((checked + 1))
|
|
if ! (cd "$FX" && ./scripts/check_logging_guardrails.sh) >"$LOG_OUT" 2>&1; then
|
|
fail "logging guardrails failed with the TRACE hot span in $LISTING: $(tail -3 "$LOG_OUT")"
|
|
fi
|
|
GET_SOURCE="rustfs/src/app/object/get.rs"
|
|
cp "$FX/$GET_SOURCE" "$FX/.get-source.saved"
|
|
"$PYTHON" - "$FX/$GET_SOURCE" <<'PY'
|
|
import sys
|
|
from pathlib import Path
|
|
path = Path(sys.argv[1])
|
|
text = path.read_text()
|
|
marker = '#[instrument(name = "execute_get_object", level = "trace", skip(self, envelope, input))]'
|
|
if text.count(marker) != 1:
|
|
sys.exit("split probe: native GET must carry exactly one TRACE span")
|
|
path.write_text(text.replace(marker, marker.replace('"trace"', '"debug"')))
|
|
PY
|
|
checked=$((checked + 1))
|
|
if (cd "$FX" && ./scripts/check_logging_guardrails.sh) >"$LOG_OUT" 2>&1; then
|
|
fail "logging guardrails passed after the native GET hot span lost TRACE"
|
|
elif ! grep -qF "execute_get_object_fields must remain TRACE-only" "$LOG_OUT"; then
|
|
fail "native GET logging guardrails failed for another reason: $(tail -3 "$LOG_OUT")"
|
|
fi
|
|
mv "$FX/.get-source.saved" "$FX/$GET_SOURCE"
|
|
"$PYTHON" - "$FX/$LISTING" <<'PY'
|
|
import sys
|
|
from pathlib import Path
|
|
path = Path(sys.argv[1])
|
|
text = path.read_text()
|
|
markers = [
|
|
f'#[instrument(level = "trace", skip({arguments}))]\n pub async fn execute_list_objects_v2('
|
|
for arguments in ("self, req", "self, envelope, input")
|
|
]
|
|
if sum(text.count(marker) for marker in markers) != 1:
|
|
sys.exit(f"split probe: {path} must carry the TRACE attribute directly on execute_list_objects_v2")
|
|
marker = next(marker for marker in markers if marker in text)
|
|
path.write_text(text.replace(marker, marker.replace('"trace"', '"debug"')))
|
|
PY
|
|
checked=$((checked + 1))
|
|
if (cd "$FX" && ./scripts/check_logging_guardrails.sh) >"$LOG_OUT" 2>&1; then
|
|
fail "logging guardrails passed after the hot span in $LISTING lost TRACE"
|
|
elif ! grep -qF "execute_list_objects_v2 must remain TRACE-only" "$LOG_OUT"; then
|
|
fail "logging guardrails failed for another reason: $(tail -3 "$LOG_OUT")"
|
|
fi
|
|
|
|
if [[ $failures -gt 0 ]]; then
|
|
echo "split guard rules probe: $failures of $checked checks failed" >&2
|
|
exit 1
|
|
fi
|
|
echo "split guard rules probe: $checked checks passed (bucket usecase $BU_STATE, multipart usecase $MU_STATE)"
|