fix(heal): retain legacy MRF responsibility after physical repair (#9391)

This commit is contained in:
cxymds authored and GitHub committed 2026-11-13 11:03:53 +08:00
1 parent ef79dcbe19
commit ce9d0b153e
8 files changed
+257 -13

No files matched your search

+18
View File
@@ -492,11 +492,29 @@ impl HealManager {
}
publish_verified_mrf_repair_events(&notice_targets, &completed_status_for_verified_events);
publish_unverified_legacy_mrf_events(&notice_targets, &completed_status_for_verified_events, &result);
// Single-object MRF attempts retain one storage result.
// Use the original result even if the report was size
// limited, without certifying independent integrity.
let unverified_repair_bytes = if completed_status == HealTaskStatus::Completed
&& matches!(&result, Err(Error::LegacyMrfUnverified { .. }))
{
task.result_items
.read()
.await
.iter()
.find(|(_, item)| item.drives_healed().is_some_and(|healed| healed > 0))
.map(|(_, item)| u64::try_from(item.object_size).unwrap_or(u64::MAX))
} else {
None
};
// update statistics
let mut stats = statistics_clone.write().await;
match completed_status {
HealTaskStatus::Completed if matches!(&result, Err(Error::LegacyMrfUnverified { .. })) => {
stats.update_unverified_task_completion();
if let Some(bytes) = unverified_repair_bytes {
stats.add_unverified_repair(bytes);
}
}
HealTaskStatus::Completed => {
stats.update_task_completion(true);
+52 -4
View File
@@ -1106,7 +1106,10 @@ impl HealStorageAPI for MockStorage {
if bucket.starts_with("legacy-ambiguity-scheduler-") {
return Err(Error::Storage(EcstoreError::DeleteRecoveryLegacyAmbiguous));
}
if bucket == "legacy-held-pending-scheduler" {
if matches!(
bucket,
"legacy-held-pending-scheduler" | "legacy-held-repaired-scheduler" | "legacy-held-repaired-oversized-scheduler"
) {
let drives = vec![rustfs_madmin::heal_commands::HealDriveInfo {
state: "ok".to_owned(),
..Default::default()
@@ -1114,8 +1117,25 @@ impl HealStorageAPI for MockStorage {
return Ok((
HealResultItem {
legacy_identity_unverified: true,
detail: rustfs_heal_contracts::heal_channel::LEGACY_OBJECT_IDENTITY_UNVERIFIED_DETAIL.to_owned(),
before: rustfs_madmin::heal_commands::Infos { drives: drives.clone() },
resolved_version_id: Some([0; 16]),
disk_count: 1,
object_size: 8192,
detail: if bucket == "legacy-held-repaired-oversized-scheduler" {
"x".repeat(MAX_COMPLETED_HEAL_RESULT_BYTES)
} else {
rustfs_heal_contracts::heal_channel::LEGACY_OBJECT_IDENTITY_UNVERIFIED_DETAIL.to_owned()
},
before: rustfs_madmin::heal_commands::Infos {
drives: vec![rustfs_madmin::heal_commands::HealDriveInfo {
state: if bucket.starts_with("legacy-held-repaired-") {
"missing"
} else {
"ok"
}
.to_owned(),
..Default::default()
}],
},
after: rustfs_madmin::heal_commands::Infos { drives },
..Default::default()
},
@@ -2171,11 +2191,32 @@ fn only_typed_mrf_result_can_pause_unverified_legacy() {
#[tokio::test(flavor = "current_thread")]
async fn mrf_legacy_completed_attempt_retains_responsibility_without_failed_statistics() {
mrf_legacy_completed_attempt_scenario(false, false).await;
}
#[tokio::test(flavor = "current_thread")]
async fn mrf_legacy_repaired_attempt_retains_responsibility_and_physical_statistics() {
mrf_legacy_completed_attempt_scenario(true, false).await;
}
#[tokio::test(flavor = "current_thread")]
async fn mrf_legacy_repair_statistics_survive_completed_result_truncation() {
mrf_legacy_completed_attempt_scenario(true, true).await;
}
async fn mrf_legacy_completed_attempt_scenario(repaired: bool, oversized_result: bool) {
use crate::heal::failure_logging::test_support::FailureLogCapture;
let capture = FailureLogCapture::default();
let _subscriber = tracing::subscriber::set_default(capture.clone());
let (target, _) = unverified_legacy_fixture("legacy-held-pending-scheduler");
let bucket = if oversized_result {
"legacy-held-repaired-oversized-scheduler"
} else if repaired {
"legacy-held-repaired-scheduler"
} else {
"legacy-held-pending-scheduler"
};
let (target, _) = unverified_legacy_fixture(bucket);
let bucket = target.bucket.clone();
let anchor = target.durable_anchor.clone().expect("durable MRF anchor");
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
@@ -2202,8 +2243,15 @@ async fn mrf_legacy_completed_attempt_retains_responsibility_without_failed_stat
assert_eq!(stats.failed_tasks, 0);
assert_eq!(stats.successful_tasks, 0);
assert_eq!(stats.total_objects_healed, 0);
assert_eq!(stats.total_bytes_healed, 0);
assert_eq!(stats.total_unverified_objects_repaired, u64::from(repaired));
assert_eq!(stats.total_unverified_bytes_repaired, if repaired { 8192 } else { 0 });
let report = manager.get_task_report(&receipt.task_id).await.expect("retained task report");
assert_eq!(report.status, HealTaskStatus::Completed);
if oversized_result {
assert!(report.result_items_truncated);
assert!(report.result_items.is_empty());
}
let outcome = report.outcome.expect("retained unknown outcome");
assert_eq!(outcome.counters.unknown, 1);
assert_eq!(outcome.counters.failed, 0);
@@ -395,6 +395,77 @@ fn empty_events_keep_dirty_flush_failure_retry_and_cleanup_gates() {
});
}
#[test]
fn legacy_park_replay_requires_committed_checkpoint() {
if isolated("legacy_park_replay_requires_committed_checkpoint") {
return;
}
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("runtime")
.block_on(async {
let (env, manager, mut runtime, anchor) = bound_runtime().await;
let original = runtime.partial_writes.checkpoint_records(intent_digest);
assert!(runtime.flush().await, "the active responsibility must be durable first");
// Losing the completed-task notice leaves the original active
// owner replayable. No physical result alone can discharge it.
let mut queue = MrfQueue::new(runtime.config.queue_capacity, runtime.config.journal_max_bytes);
let replay = replay_into(&manager, &mut queue, &mut None).await;
assert_eq!(replay.partial_writes.len(), original.len());
assert!(replay.partial_writes.iter().all(|(_, record)| {
!matches!(
record.as_ref().expect("active companion").state,
partial_write::ResponsibilityState::HeldUnverifiedLegacy { .. }
)
}));
note_mrf_unverified_legacy(MrfUnverifiedLegacyEvent { anchor: anchor.clone() });
runtime.discharge_durable_replay_anchors();
assert_eq!(runtime.partial_writes.lifecycle_summary().held_count, 1);
let paths = env
.disk_paths
.iter()
.map(|disk| (disk.join(".rustfs.sys"), disk.join(".mrf-park-sys-backup")))
.collect::<Vec<_>>();
for (path, backup) in &paths {
fs::rename(path, backup).expect("preserve committed active checkpoint");
fs::write(path, b"block held checkpoint publication").expect("inject publication failure");
}
let persisted = runtime.flush().await;
for (path, backup) in &paths {
fs::remove_file(path).expect("remove publication blocker");
fs::rename(backup, path).expect("restore committed active checkpoint");
}
assert!(!persisted);
assert!(runtime.dirty);
let mut queue = MrfQueue::new(runtime.config.queue_capacity, runtime.config.journal_max_bytes);
let replay = replay_into(&manager, &mut queue, &mut None).await;
let mut restored = replay
.partial_writes
.into_iter()
.map(|(_, record)| record.expect("previous active companion"))
.collect::<Vec<_>>();
let mut original = original;
restored.sort_by_key(|record| record.responsibility_id);
original.sort_by_key(|record| record.responsibility_id);
assert_eq!(restored, original, "a crash before commit may conservatively repeat Deep");
assert!(runtime.flush().await, "commit the held state after storage recovers");
let mut queue = MrfQueue::new(runtime.config.queue_capacity, runtime.config.journal_max_bytes);
let replay = replay_into(&manager, &mut queue, &mut None).await;
assert_eq!(replay.partial_writes.len(), original.len());
assert!(replay.partial_writes.iter().any(|(_, record)| {
matches!(
record.as_ref().expect("committed held companion").state,
partial_write::ResponsibilityState::HeldUnverifiedLegacy { .. }
)
}));
assert!(runtime.retained_replay_journal(), "parking never releases responsibility");
});
}
#[test]
fn empty_events_real_consumer_stop_retains_unresolved_journal() {
if isolated("empty_events_real_consumer_stop_retains_unresolved_journal") {
+25 -1
View File
@@ -509,6 +509,12 @@ pub struct HealStatistics {
/// Completed attempts without independent verification; not the current durable MRF backlog.
#[serde(default)]
pub unverified_tasks: u64,
/// Physical repair attempts for unverified legacy objects, not verified outcomes or unique objects.
#[serde(default)]
pub total_unverified_objects_repaired: u64,
/// Logical object bytes in those repair attempts, not physical disk or network bytes.
#[serde(default)]
pub total_unverified_bytes_repaired: u64,
/// Running tasks
pub running_tasks: u64,
/// Total healed objects
@@ -532,6 +538,8 @@ impl HealStatistics {
successful_tasks: 0,
failed_tasks: 0,
unverified_tasks: 0,
total_unverified_objects_repaired: 0,
total_unverified_bytes_repaired: 0,
running_tasks: 0,
total_objects_healed: 0,
total_bytes_healed: 0,
@@ -558,6 +566,11 @@ impl HealStatistics {
self.last_update_time = SystemTime::now();
}
pub(crate) fn add_unverified_repair(&mut self, bytes: u64) {
self.total_unverified_objects_repaired = self.total_unverified_objects_repaired.saturating_add(1);
self.total_unverified_bytes_repaired = self.total_unverified_bytes_repaired.saturating_add(bytes);
}
pub fn add_healed_objects(&mut self, count: u64, bytes: u64) {
self.total_objects_healed += count;
self.total_bytes_healed += bytes;
@@ -928,15 +941,26 @@ mod tests {
fn unverified_statistics_preserve_old_snapshot_compatibility() {
let mut stats = HealStatistics::new();
stats.update_unverified_task_completion();
stats.add_unverified_repair(8192);
assert_eq!(stats.unverified_tasks, 1);
assert_eq!(stats.successful_tasks, 0);
assert_eq!(stats.failed_tasks, 0);
let mut wire = serde_json::to_value(&stats).expect("serialize task statistics");
let current: HealStatistics = serde_json::from_value(wire.clone()).expect("read new statistics");
assert_eq!(current.unverified_tasks, 1);
wire.as_object_mut().expect("statistics object").remove("unverified_tasks");
assert_eq!(current.total_unverified_objects_repaired, 1);
assert_eq!(current.total_unverified_bytes_repaired, 8192);
for field in [
"unverified_tasks",
"total_unverified_objects_repaired",
"total_unverified_bytes_repaired",
] {
wire.as_object_mut().expect("statistics object").remove(field);
}
let old: HealStatistics = serde_json::from_value(wire).expect("read statistics without new field");
assert_eq!(old.unverified_tasks, 0);
assert_eq!(old.total_unverified_objects_repaired, 0);
assert_eq!(old.total_unverified_bytes_repaired, 0);
}
#[test]
+24 -5
View File
@@ -514,17 +514,36 @@ impl HealTask {
let receipt_missing = storage_result.receipt.is_none();
if !self.record_verified_storage_receipt(&expected, storage_result.receipt).await {
let ok_drive_state = DriveState::Ok.to_string();
let healthy_legacy_without_repair = receipt_missing
let selected_version_matches = storage_result.item.resolved_version_id.is_some_and(|resolved| {
let requested = version_id.filter(|version| !version.is_empty());
match requested {
Some(requested) => Uuid::parse_str(requested).is_ok_and(|version| version.as_bytes() == &resolved),
None => resolved == [0; 16],
}
});
// A completed legacy repair can restore redundancy without proving
// original content. Keep the same durable responsibility as an
// already-healthy Deep result, including when this attempt repaired
// a shard. Storage errors and invalid receipts never qualify.
let healthy_legacy = receipt_missing
&& storage_result.item.legacy_identity_unverified
&& storage_result.item.drives_healed() == Some(0)
&& !storage_result.item.after.drives.is_empty()
&& selected_version_matches
&& storage_result.item.drives_healed().is_some()
&& storage_result.item.after.drives.len() == storage_result.item.disk_count
&& storage_result
.item
.before
.drives
.iter()
.zip(&storage_result.item.after.drives)
.all(|(before, after)| before.endpoint == after.endpoint)
&& storage_result
.item
.after
.drives
.iter()
.all(|drive| drive.state == ok_drive_state);
if healthy_legacy_without_repair {
if healthy_legacy {
self.outcome.write().await.record(HealObjectOutcome {
deep_check_observed: false,
quorum_selected_data_dir: None,
@@ -534,7 +553,7 @@ impl HealTask {
});
}
self.record_result_item(storage_result.item).await;
return Err(if healthy_legacy_without_repair {
return Err(if healthy_legacy {
Error::LegacyMrfUnverified {
target: format!("{bucket}/{object}"),
}
+54 -3
View File
@@ -3227,6 +3227,8 @@ fn healthy_legacy_heal_item() -> HealResultItem {
}];
HealResultItem {
legacy_identity_unverified: true,
resolved_version_id: Some([0; 16]),
disk_count: 1,
detail: rustfs_heal_contracts::heal_channel::LEGACY_OBJECT_IDENTITY_UNVERIFIED_DETAIL.to_owned(),
before: Infos { drives: drives.clone() },
after: Infos { drives },
@@ -3234,6 +3236,34 @@ fn healthy_legacy_heal_item() -> HealResultItem {
}
}
#[tokio::test]
async fn mrf_legacy_repair_completes_unverified_without_second_deep() {
let mut item = healthy_legacy_heal_item();
item.before.drives[0].state = "missing".to_owned();
item.object_size = 8192;
let (task, storage) = mrf_task_for_item("legacy-repaired", item);
let error = task
.execute()
.await
.expect_err("physical repair cannot certify legacy identity");
assert!(matches!(error, Error::LegacyMrfUnverified { .. }), "{error:?}");
assert_eq!(task.get_status().await, HealTaskStatus::Completed);
let outcome = task.get_outcome().await;
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
assert_eq!(outcome.counters.unknown, 1);
assert_eq!(outcome.counters.healed, 0);
assert_eq!(outcome.counters.failed, 0);
assert!(matches!(outcome.objects[0].disposition, HealObjectDisposition::Unknown));
let items = task.get_seqed_result_items().await;
assert_eq!(items.len(), 1);
assert_eq!(items[0].1.drives_healed(), Some(1));
assert_eq!(items[0].1.object_size, 8192);
let opts = storage.object_heal_opts.lock().expect("physical repair options");
assert_eq!(opts.len(), 1, "the first completed repair must supply the terminal classification");
assert_eq!(opts[0].scan_mode, HealScanMode::Deep);
}
fn mrf_task_for_item(object: &str, item: HealResultItem) -> (HealTask, Arc<MockStorage>) {
let incarnation = Uuid::new_v4();
let storage = Arc::new(MockStorage {
@@ -3343,11 +3373,18 @@ async fn mrf_legacy_pause_requires_storage_owned_healthy_deep_result() {
"empty",
"unhealthy",
"unaligned",
"wrong_endpoint",
"wrong_disk_count",
"wrong_version",
"unknown_version",
"versioned",
"versioned_mismatch",
"receipt",
"storage_error",
"dry_run",
] {
let mut item = healthy_legacy_heal_item();
let requested_version = Uuid::new_v4();
match case {
"qualified" => item.detail = "updated diagnostic wording".to_owned(),
"text_only" => item.legacy_identity_unverified = false,
@@ -3361,9 +3398,20 @@ async fn mrf_legacy_pause_requires_storage_owned_healthy_deep_result() {
item.after.drives[0].state = "missing".to_owned();
}
"unaligned" => item.before.drives.clear(),
"wrong_endpoint" => item.before.drives[0].endpoint = "another-disk".to_owned(),
"wrong_disk_count" => item.disk_count = 4,
"wrong_version" => item.resolved_version_id = Some(*Uuid::new_v4().as_bytes()),
"unknown_version" => item.resolved_version_id = None,
"versioned" => item.resolved_version_id = Some(*requested_version.as_bytes()),
"versioned_mismatch" => item.resolved_version_id = Some(*Uuid::new_v4().as_bytes()),
_ => {}
}
let (mut task, storage) = mrf_task_for_item("object", item);
if matches!(case, "versioned" | "versioned_mismatch")
&& let HealType::Object { version_id, .. } = &mut task.heal_type
{
*version_id = Some(requested_version.to_string());
}
if case == "receipt" {
storage.heal_object_receipts.lock().expect("receipt fixture").insert(
"object".to_owned(),
@@ -3383,13 +3431,16 @@ async fn mrf_legacy_pause_requires_storage_owned_healthy_deep_result() {
.execute()
.await
.expect_err("legacy result cannot discharge a durable responsibility");
assert_eq!(matches!(error, Error::LegacyMrfUnverified { .. }), case == "qualified", "case={case}");
let qualifies = matches!(case, "qualified" | "repaired" | "versioned");
assert_eq!(matches!(error, Error::LegacyMrfUnverified { .. }), qualifies, "case={case}");
let outcome = task.get_outcome().await;
assert_eq!(outcome.counters.healed, 0);
if case == "qualified" {
if qualifies {
assert_eq!(task.get_status().await, HealTaskStatus::Completed);
assert_eq!(outcome.counters.unknown, 1);
assert_eq!(outcome.objects[0].detail.as_deref(), Some("updated diagnostic wording"));
if case == "qualified" {
assert_eq!(outcome.objects[0].detail.as_deref(), Some("updated diagnostic wording"));
}
let opts = storage.object_heal_opts.lock().expect("storage options");
assert_eq!(opts.len(), 1);
assert_eq!(opts[0].scan_mode, HealScanMode::Deep);
@@ -1004,6 +1004,15 @@ async fn partial_write_sigkill_replay_scenario(protected: bool) {
.await,
"legacy replay attempts must finish"
);
let stats = manager.get_statistics().await;
assert_eq!(stats.failed_tasks, 0, "the first physical legacy repair must not be a failed attempt");
assert_eq!(
stats.unverified_tasks, 1,
"no second Deep is needed to classify the repaired legacy object"
);
assert_eq!(stats.total_unverified_objects_repaired, 1);
assert_eq!(stats.total_unverified_bytes_repaired, 36);
assert_eq!(stats.total_objects_healed, 0, "physical repair does not prove legacy identity");
// Cover two MRF admission backoff periods so a missed hold notice
// cannot pass merely because its task finished between polls.
let retry_window = tokio::time::Instant::now() + Duration::from_secs(12);
@@ -65,6 +65,10 @@ queue does not imply that these persistent obligations have been cleared.
After a qualified all-healthy Deep check, the attempt is `Completed` with an `Unknown` object outcome; it is counted in `unverified_tasks`, not `successful_tasks` or `failed_tasks`. This cumulative attempt count is distinct from the current `held_unverified_legacy` responsibility count. The scheduler reports `verification_unavailable`, and the durable obligation remains subject to the existing explicit recheck or audited operator disposition. Execution completion never supplies a missing integrity receipt.
This also applies when the same Deep attempt has just reconstructed missing legacy shards: successful storage commits and a complete, aligned, all-healthy report for the selected version permit an unverified completion without another Deep pass. Storage failures, incomplete commits, lost locks, invalid receipts and protected-object verification failures do not qualify. `total_unverified_objects_repaired` and `total_unverified_bytes_repaired` record physical repair attempts separately from verified outcomes; bytes are logical object sizes, not disk or network traffic, and repeated repairs of the same object are separate attempts. An already-healthy check adds no physical repair count.
Attempt completion, an in-memory hold and a durable hold are separate steps. The hold survives restart after its companion checkpoint commits. A crash before that commit, a lost completion notice or an unavailable publication result can conservatively replay the earlier active responsibility and repeat Deep. Preserve the prior checkpoint and responsibility until a committed publication or exact readback confirms the transition; no retry or execution completion supplies independent verification.
Partial-write MRF replay uses Deep verification so a protected object's repair
can discharge its obligation after verifying the payload. This adds full-object
read work to those background attempts, including healthy replay targets.