mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-20 13:19:34 +08:00
fix(heal): retain legacy MRF responsibility after physical repair (#9391)
This commit is contained in:
1 parent
ef79dcbe19
commit
ce9d0b153e
8 files changed
+257
-13
No files matched your search
@@ -492,11 +492,29 @@ impl HealManager {
|
||||
}
|
||||
publish_verified_mrf_repair_events(¬ice_targets, &completed_status_for_verified_events);
|
||||
publish_unverified_legacy_mrf_events(¬ice_targets, &completed_status_for_verified_events, &result);
|
||||
// Single-object MRF attempts retain one storage result.
|
||||
// Use the original result even if the report was size
|
||||
// limited, without certifying independent integrity.
|
||||
let unverified_repair_bytes = if completed_status == HealTaskStatus::Completed
|
||||
&& matches!(&result, Err(Error::LegacyMrfUnverified { .. }))
|
||||
{
|
||||
task.result_items
|
||||
.read()
|
||||
.await
|
||||
.iter()
|
||||
.find(|(_, item)| item.drives_healed().is_some_and(|healed| healed > 0))
|
||||
.map(|(_, item)| u64::try_from(item.object_size).unwrap_or(u64::MAX))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// update statistics
|
||||
let mut stats = statistics_clone.write().await;
|
||||
match completed_status {
|
||||
HealTaskStatus::Completed if matches!(&result, Err(Error::LegacyMrfUnverified { .. })) => {
|
||||
stats.update_unverified_task_completion();
|
||||
if let Some(bytes) = unverified_repair_bytes {
|
||||
stats.add_unverified_repair(bytes);
|
||||
}
|
||||
}
|
||||
HealTaskStatus::Completed => {
|
||||
stats.update_task_completion(true);
|
||||
|
||||
@@ -1106,7 +1106,10 @@ impl HealStorageAPI for MockStorage {
|
||||
if bucket.starts_with("legacy-ambiguity-scheduler-") {
|
||||
return Err(Error::Storage(EcstoreError::DeleteRecoveryLegacyAmbiguous));
|
||||
}
|
||||
if bucket == "legacy-held-pending-scheduler" {
|
||||
if matches!(
|
||||
bucket,
|
||||
"legacy-held-pending-scheduler" | "legacy-held-repaired-scheduler" | "legacy-held-repaired-oversized-scheduler"
|
||||
) {
|
||||
let drives = vec![rustfs_madmin::heal_commands::HealDriveInfo {
|
||||
state: "ok".to_owned(),
|
||||
..Default::default()
|
||||
@@ -1114,8 +1117,25 @@ impl HealStorageAPI for MockStorage {
|
||||
return Ok((
|
||||
HealResultItem {
|
||||
legacy_identity_unverified: true,
|
||||
detail: rustfs_heal_contracts::heal_channel::LEGACY_OBJECT_IDENTITY_UNVERIFIED_DETAIL.to_owned(),
|
||||
before: rustfs_madmin::heal_commands::Infos { drives: drives.clone() },
|
||||
resolved_version_id: Some([0; 16]),
|
||||
disk_count: 1,
|
||||
object_size: 8192,
|
||||
detail: if bucket == "legacy-held-repaired-oversized-scheduler" {
|
||||
"x".repeat(MAX_COMPLETED_HEAL_RESULT_BYTES)
|
||||
} else {
|
||||
rustfs_heal_contracts::heal_channel::LEGACY_OBJECT_IDENTITY_UNVERIFIED_DETAIL.to_owned()
|
||||
},
|
||||
before: rustfs_madmin::heal_commands::Infos {
|
||||
drives: vec![rustfs_madmin::heal_commands::HealDriveInfo {
|
||||
state: if bucket.starts_with("legacy-held-repaired-") {
|
||||
"missing"
|
||||
} else {
|
||||
"ok"
|
||||
}
|
||||
.to_owned(),
|
||||
..Default::default()
|
||||
}],
|
||||
},
|
||||
after: rustfs_madmin::heal_commands::Infos { drives },
|
||||
..Default::default()
|
||||
},
|
||||
@@ -2171,11 +2191,32 @@ fn only_typed_mrf_result_can_pause_unverified_legacy() {
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn mrf_legacy_completed_attempt_retains_responsibility_without_failed_statistics() {
|
||||
mrf_legacy_completed_attempt_scenario(false, false).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn mrf_legacy_repaired_attempt_retains_responsibility_and_physical_statistics() {
|
||||
mrf_legacy_completed_attempt_scenario(true, false).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn mrf_legacy_repair_statistics_survive_completed_result_truncation() {
|
||||
mrf_legacy_completed_attempt_scenario(true, true).await;
|
||||
}
|
||||
|
||||
async fn mrf_legacy_completed_attempt_scenario(repaired: bool, oversized_result: bool) {
|
||||
use crate::heal::failure_logging::test_support::FailureLogCapture;
|
||||
|
||||
let capture = FailureLogCapture::default();
|
||||
let _subscriber = tracing::subscriber::set_default(capture.clone());
|
||||
let (target, _) = unverified_legacy_fixture("legacy-held-pending-scheduler");
|
||||
let bucket = if oversized_result {
|
||||
"legacy-held-repaired-oversized-scheduler"
|
||||
} else if repaired {
|
||||
"legacy-held-repaired-scheduler"
|
||||
} else {
|
||||
"legacy-held-pending-scheduler"
|
||||
};
|
||||
let (target, _) = unverified_legacy_fixture(bucket);
|
||||
let bucket = target.bucket.clone();
|
||||
let anchor = target.durable_anchor.clone().expect("durable MRF anchor");
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
@@ -2202,8 +2243,15 @@ async fn mrf_legacy_completed_attempt_retains_responsibility_without_failed_stat
|
||||
assert_eq!(stats.failed_tasks, 0);
|
||||
assert_eq!(stats.successful_tasks, 0);
|
||||
assert_eq!(stats.total_objects_healed, 0);
|
||||
assert_eq!(stats.total_bytes_healed, 0);
|
||||
assert_eq!(stats.total_unverified_objects_repaired, u64::from(repaired));
|
||||
assert_eq!(stats.total_unverified_bytes_repaired, if repaired { 8192 } else { 0 });
|
||||
let report = manager.get_task_report(&receipt.task_id).await.expect("retained task report");
|
||||
assert_eq!(report.status, HealTaskStatus::Completed);
|
||||
if oversized_result {
|
||||
assert!(report.result_items_truncated);
|
||||
assert!(report.result_items.is_empty());
|
||||
}
|
||||
let outcome = report.outcome.expect("retained unknown outcome");
|
||||
assert_eq!(outcome.counters.unknown, 1);
|
||||
assert_eq!(outcome.counters.failed, 0);
|
||||
|
||||
@@ -395,6 +395,77 @@ fn empty_events_keep_dirty_flush_failure_retry_and_cleanup_gates() {
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_park_replay_requires_committed_checkpoint() {
|
||||
if isolated("legacy_park_replay_requires_committed_checkpoint") {
|
||||
return;
|
||||
}
|
||||
tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("runtime")
|
||||
.block_on(async {
|
||||
let (env, manager, mut runtime, anchor) = bound_runtime().await;
|
||||
let original = runtime.partial_writes.checkpoint_records(intent_digest);
|
||||
assert!(runtime.flush().await, "the active responsibility must be durable first");
|
||||
|
||||
// Losing the completed-task notice leaves the original active
|
||||
// owner replayable. No physical result alone can discharge it.
|
||||
let mut queue = MrfQueue::new(runtime.config.queue_capacity, runtime.config.journal_max_bytes);
|
||||
let replay = replay_into(&manager, &mut queue, &mut None).await;
|
||||
assert_eq!(replay.partial_writes.len(), original.len());
|
||||
assert!(replay.partial_writes.iter().all(|(_, record)| {
|
||||
!matches!(
|
||||
record.as_ref().expect("active companion").state,
|
||||
partial_write::ResponsibilityState::HeldUnverifiedLegacy { .. }
|
||||
)
|
||||
}));
|
||||
|
||||
note_mrf_unverified_legacy(MrfUnverifiedLegacyEvent { anchor: anchor.clone() });
|
||||
runtime.discharge_durable_replay_anchors();
|
||||
assert_eq!(runtime.partial_writes.lifecycle_summary().held_count, 1);
|
||||
let paths = env
|
||||
.disk_paths
|
||||
.iter()
|
||||
.map(|disk| (disk.join(".rustfs.sys"), disk.join(".mrf-park-sys-backup")))
|
||||
.collect::<Vec<_>>();
|
||||
for (path, backup) in &paths {
|
||||
fs::rename(path, backup).expect("preserve committed active checkpoint");
|
||||
fs::write(path, b"block held checkpoint publication").expect("inject publication failure");
|
||||
}
|
||||
let persisted = runtime.flush().await;
|
||||
for (path, backup) in &paths {
|
||||
fs::remove_file(path).expect("remove publication blocker");
|
||||
fs::rename(backup, path).expect("restore committed active checkpoint");
|
||||
}
|
||||
assert!(!persisted);
|
||||
assert!(runtime.dirty);
|
||||
let mut queue = MrfQueue::new(runtime.config.queue_capacity, runtime.config.journal_max_bytes);
|
||||
let replay = replay_into(&manager, &mut queue, &mut None).await;
|
||||
let mut restored = replay
|
||||
.partial_writes
|
||||
.into_iter()
|
||||
.map(|(_, record)| record.expect("previous active companion"))
|
||||
.collect::<Vec<_>>();
|
||||
let mut original = original;
|
||||
restored.sort_by_key(|record| record.responsibility_id);
|
||||
original.sort_by_key(|record| record.responsibility_id);
|
||||
assert_eq!(restored, original, "a crash before commit may conservatively repeat Deep");
|
||||
|
||||
assert!(runtime.flush().await, "commit the held state after storage recovers");
|
||||
let mut queue = MrfQueue::new(runtime.config.queue_capacity, runtime.config.journal_max_bytes);
|
||||
let replay = replay_into(&manager, &mut queue, &mut None).await;
|
||||
assert_eq!(replay.partial_writes.len(), original.len());
|
||||
assert!(replay.partial_writes.iter().any(|(_, record)| {
|
||||
matches!(
|
||||
record.as_ref().expect("committed held companion").state,
|
||||
partial_write::ResponsibilityState::HeldUnverifiedLegacy { .. }
|
||||
)
|
||||
}));
|
||||
assert!(runtime.retained_replay_journal(), "parking never releases responsibility");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_events_real_consumer_stop_retains_unresolved_journal() {
|
||||
if isolated("empty_events_real_consumer_stop_retains_unresolved_journal") {
|
||||
|
||||
@@ -509,6 +509,12 @@ pub struct HealStatistics {
|
||||
/// Completed attempts without independent verification; not the current durable MRF backlog.
|
||||
#[serde(default)]
|
||||
pub unverified_tasks: u64,
|
||||
/// Physical repair attempts for unverified legacy objects, not verified outcomes or unique objects.
|
||||
#[serde(default)]
|
||||
pub total_unverified_objects_repaired: u64,
|
||||
/// Logical object bytes in those repair attempts, not physical disk or network bytes.
|
||||
#[serde(default)]
|
||||
pub total_unverified_bytes_repaired: u64,
|
||||
/// Running tasks
|
||||
pub running_tasks: u64,
|
||||
/// Total healed objects
|
||||
@@ -532,6 +538,8 @@ impl HealStatistics {
|
||||
successful_tasks: 0,
|
||||
failed_tasks: 0,
|
||||
unverified_tasks: 0,
|
||||
total_unverified_objects_repaired: 0,
|
||||
total_unverified_bytes_repaired: 0,
|
||||
running_tasks: 0,
|
||||
total_objects_healed: 0,
|
||||
total_bytes_healed: 0,
|
||||
@@ -558,6 +566,11 @@ impl HealStatistics {
|
||||
self.last_update_time = SystemTime::now();
|
||||
}
|
||||
|
||||
pub(crate) fn add_unverified_repair(&mut self, bytes: u64) {
|
||||
self.total_unverified_objects_repaired = self.total_unverified_objects_repaired.saturating_add(1);
|
||||
self.total_unverified_bytes_repaired = self.total_unverified_bytes_repaired.saturating_add(bytes);
|
||||
}
|
||||
|
||||
pub fn add_healed_objects(&mut self, count: u64, bytes: u64) {
|
||||
self.total_objects_healed += count;
|
||||
self.total_bytes_healed += bytes;
|
||||
@@ -928,15 +941,26 @@ mod tests {
|
||||
fn unverified_statistics_preserve_old_snapshot_compatibility() {
|
||||
let mut stats = HealStatistics::new();
|
||||
stats.update_unverified_task_completion();
|
||||
stats.add_unverified_repair(8192);
|
||||
assert_eq!(stats.unverified_tasks, 1);
|
||||
assert_eq!(stats.successful_tasks, 0);
|
||||
assert_eq!(stats.failed_tasks, 0);
|
||||
let mut wire = serde_json::to_value(&stats).expect("serialize task statistics");
|
||||
let current: HealStatistics = serde_json::from_value(wire.clone()).expect("read new statistics");
|
||||
assert_eq!(current.unverified_tasks, 1);
|
||||
wire.as_object_mut().expect("statistics object").remove("unverified_tasks");
|
||||
assert_eq!(current.total_unverified_objects_repaired, 1);
|
||||
assert_eq!(current.total_unverified_bytes_repaired, 8192);
|
||||
for field in [
|
||||
"unverified_tasks",
|
||||
"total_unverified_objects_repaired",
|
||||
"total_unverified_bytes_repaired",
|
||||
] {
|
||||
wire.as_object_mut().expect("statistics object").remove(field);
|
||||
}
|
||||
let old: HealStatistics = serde_json::from_value(wire).expect("read statistics without new field");
|
||||
assert_eq!(old.unverified_tasks, 0);
|
||||
assert_eq!(old.total_unverified_objects_repaired, 0);
|
||||
assert_eq!(old.total_unverified_bytes_repaired, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -514,17 +514,36 @@ impl HealTask {
|
||||
let receipt_missing = storage_result.receipt.is_none();
|
||||
if !self.record_verified_storage_receipt(&expected, storage_result.receipt).await {
|
||||
let ok_drive_state = DriveState::Ok.to_string();
|
||||
let healthy_legacy_without_repair = receipt_missing
|
||||
let selected_version_matches = storage_result.item.resolved_version_id.is_some_and(|resolved| {
|
||||
let requested = version_id.filter(|version| !version.is_empty());
|
||||
match requested {
|
||||
Some(requested) => Uuid::parse_str(requested).is_ok_and(|version| version.as_bytes() == &resolved),
|
||||
None => resolved == [0; 16],
|
||||
}
|
||||
});
|
||||
// A completed legacy repair can restore redundancy without proving
|
||||
// original content. Keep the same durable responsibility as an
|
||||
// already-healthy Deep result, including when this attempt repaired
|
||||
// a shard. Storage errors and invalid receipts never qualify.
|
||||
let healthy_legacy = receipt_missing
|
||||
&& storage_result.item.legacy_identity_unverified
|
||||
&& storage_result.item.drives_healed() == Some(0)
|
||||
&& !storage_result.item.after.drives.is_empty()
|
||||
&& selected_version_matches
|
||||
&& storage_result.item.drives_healed().is_some()
|
||||
&& storage_result.item.after.drives.len() == storage_result.item.disk_count
|
||||
&& storage_result
|
||||
.item
|
||||
.before
|
||||
.drives
|
||||
.iter()
|
||||
.zip(&storage_result.item.after.drives)
|
||||
.all(|(before, after)| before.endpoint == after.endpoint)
|
||||
&& storage_result
|
||||
.item
|
||||
.after
|
||||
.drives
|
||||
.iter()
|
||||
.all(|drive| drive.state == ok_drive_state);
|
||||
if healthy_legacy_without_repair {
|
||||
if healthy_legacy {
|
||||
self.outcome.write().await.record(HealObjectOutcome {
|
||||
deep_check_observed: false,
|
||||
quorum_selected_data_dir: None,
|
||||
@@ -534,7 +553,7 @@ impl HealTask {
|
||||
});
|
||||
}
|
||||
self.record_result_item(storage_result.item).await;
|
||||
return Err(if healthy_legacy_without_repair {
|
||||
return Err(if healthy_legacy {
|
||||
Error::LegacyMrfUnverified {
|
||||
target: format!("{bucket}/{object}"),
|
||||
}
|
||||
|
||||
@@ -3227,6 +3227,8 @@ fn healthy_legacy_heal_item() -> HealResultItem {
|
||||
}];
|
||||
HealResultItem {
|
||||
legacy_identity_unverified: true,
|
||||
resolved_version_id: Some([0; 16]),
|
||||
disk_count: 1,
|
||||
detail: rustfs_heal_contracts::heal_channel::LEGACY_OBJECT_IDENTITY_UNVERIFIED_DETAIL.to_owned(),
|
||||
before: Infos { drives: drives.clone() },
|
||||
after: Infos { drives },
|
||||
@@ -3234,6 +3236,34 @@ fn healthy_legacy_heal_item() -> HealResultItem {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_legacy_repair_completes_unverified_without_second_deep() {
|
||||
let mut item = healthy_legacy_heal_item();
|
||||
item.before.drives[0].state = "missing".to_owned();
|
||||
item.object_size = 8192;
|
||||
let (task, storage) = mrf_task_for_item("legacy-repaired", item);
|
||||
|
||||
let error = task
|
||||
.execute()
|
||||
.await
|
||||
.expect_err("physical repair cannot certify legacy identity");
|
||||
assert!(matches!(error, Error::LegacyMrfUnverified { .. }), "{error:?}");
|
||||
assert_eq!(task.get_status().await, HealTaskStatus::Completed);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
|
||||
assert_eq!(outcome.counters.unknown, 1);
|
||||
assert_eq!(outcome.counters.healed, 0);
|
||||
assert_eq!(outcome.counters.failed, 0);
|
||||
assert!(matches!(outcome.objects[0].disposition, HealObjectDisposition::Unknown));
|
||||
let items = task.get_seqed_result_items().await;
|
||||
assert_eq!(items.len(), 1);
|
||||
assert_eq!(items[0].1.drives_healed(), Some(1));
|
||||
assert_eq!(items[0].1.object_size, 8192);
|
||||
let opts = storage.object_heal_opts.lock().expect("physical repair options");
|
||||
assert_eq!(opts.len(), 1, "the first completed repair must supply the terminal classification");
|
||||
assert_eq!(opts[0].scan_mode, HealScanMode::Deep);
|
||||
}
|
||||
|
||||
fn mrf_task_for_item(object: &str, item: HealResultItem) -> (HealTask, Arc<MockStorage>) {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
@@ -3343,11 +3373,18 @@ async fn mrf_legacy_pause_requires_storage_owned_healthy_deep_result() {
|
||||
"empty",
|
||||
"unhealthy",
|
||||
"unaligned",
|
||||
"wrong_endpoint",
|
||||
"wrong_disk_count",
|
||||
"wrong_version",
|
||||
"unknown_version",
|
||||
"versioned",
|
||||
"versioned_mismatch",
|
||||
"receipt",
|
||||
"storage_error",
|
||||
"dry_run",
|
||||
] {
|
||||
let mut item = healthy_legacy_heal_item();
|
||||
let requested_version = Uuid::new_v4();
|
||||
match case {
|
||||
"qualified" => item.detail = "updated diagnostic wording".to_owned(),
|
||||
"text_only" => item.legacy_identity_unverified = false,
|
||||
@@ -3361,9 +3398,20 @@ async fn mrf_legacy_pause_requires_storage_owned_healthy_deep_result() {
|
||||
item.after.drives[0].state = "missing".to_owned();
|
||||
}
|
||||
"unaligned" => item.before.drives.clear(),
|
||||
"wrong_endpoint" => item.before.drives[0].endpoint = "another-disk".to_owned(),
|
||||
"wrong_disk_count" => item.disk_count = 4,
|
||||
"wrong_version" => item.resolved_version_id = Some(*Uuid::new_v4().as_bytes()),
|
||||
"unknown_version" => item.resolved_version_id = None,
|
||||
"versioned" => item.resolved_version_id = Some(*requested_version.as_bytes()),
|
||||
"versioned_mismatch" => item.resolved_version_id = Some(*Uuid::new_v4().as_bytes()),
|
||||
_ => {}
|
||||
}
|
||||
let (mut task, storage) = mrf_task_for_item("object", item);
|
||||
if matches!(case, "versioned" | "versioned_mismatch")
|
||||
&& let HealType::Object { version_id, .. } = &mut task.heal_type
|
||||
{
|
||||
*version_id = Some(requested_version.to_string());
|
||||
}
|
||||
if case == "receipt" {
|
||||
storage.heal_object_receipts.lock().expect("receipt fixture").insert(
|
||||
"object".to_owned(),
|
||||
@@ -3383,13 +3431,16 @@ async fn mrf_legacy_pause_requires_storage_owned_healthy_deep_result() {
|
||||
.execute()
|
||||
.await
|
||||
.expect_err("legacy result cannot discharge a durable responsibility");
|
||||
assert_eq!(matches!(error, Error::LegacyMrfUnverified { .. }), case == "qualified", "case={case}");
|
||||
let qualifies = matches!(case, "qualified" | "repaired" | "versioned");
|
||||
assert_eq!(matches!(error, Error::LegacyMrfUnverified { .. }), qualifies, "case={case}");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.healed, 0);
|
||||
if case == "qualified" {
|
||||
if qualifies {
|
||||
assert_eq!(task.get_status().await, HealTaskStatus::Completed);
|
||||
assert_eq!(outcome.counters.unknown, 1);
|
||||
assert_eq!(outcome.objects[0].detail.as_deref(), Some("updated diagnostic wording"));
|
||||
if case == "qualified" {
|
||||
assert_eq!(outcome.objects[0].detail.as_deref(), Some("updated diagnostic wording"));
|
||||
}
|
||||
let opts = storage.object_heal_opts.lock().expect("storage options");
|
||||
assert_eq!(opts.len(), 1);
|
||||
assert_eq!(opts[0].scan_mode, HealScanMode::Deep);
|
||||
|
||||
@@ -1004,6 +1004,15 @@ async fn partial_write_sigkill_replay_scenario(protected: bool) {
|
||||
.await,
|
||||
"legacy replay attempts must finish"
|
||||
);
|
||||
let stats = manager.get_statistics().await;
|
||||
assert_eq!(stats.failed_tasks, 0, "the first physical legacy repair must not be a failed attempt");
|
||||
assert_eq!(
|
||||
stats.unverified_tasks, 1,
|
||||
"no second Deep is needed to classify the repaired legacy object"
|
||||
);
|
||||
assert_eq!(stats.total_unverified_objects_repaired, 1);
|
||||
assert_eq!(stats.total_unverified_bytes_repaired, 36);
|
||||
assert_eq!(stats.total_objects_healed, 0, "physical repair does not prove legacy identity");
|
||||
// Cover two MRF admission backoff periods so a missed hold notice
|
||||
// cannot pass merely because its task finished between polls.
|
||||
let retry_window = tokio::time::Instant::now() + Duration::from_secs(12);
|
||||
|
||||
@@ -65,6 +65,10 @@ queue does not imply that these persistent obligations have been cleared.
|
||||
|
||||
After a qualified all-healthy Deep check, the attempt is `Completed` with an `Unknown` object outcome; it is counted in `unverified_tasks`, not `successful_tasks` or `failed_tasks`. This cumulative attempt count is distinct from the current `held_unverified_legacy` responsibility count. The scheduler reports `verification_unavailable`, and the durable obligation remains subject to the existing explicit recheck or audited operator disposition. Execution completion never supplies a missing integrity receipt.
|
||||
|
||||
This also applies when the same Deep attempt has just reconstructed missing legacy shards: successful storage commits and a complete, aligned, all-healthy report for the selected version permit an unverified completion without another Deep pass. Storage failures, incomplete commits, lost locks, invalid receipts and protected-object verification failures do not qualify. `total_unverified_objects_repaired` and `total_unverified_bytes_repaired` record physical repair attempts separately from verified outcomes; bytes are logical object sizes, not disk or network traffic, and repeated repairs of the same object are separate attempts. An already-healthy check adds no physical repair count.
|
||||
|
||||
Attempt completion, an in-memory hold and a durable hold are separate steps. The hold survives restart after its companion checkpoint commits. A crash before that commit, a lost completion notice or an unavailable publication result can conservatively replay the earlier active responsibility and repeat Deep. Preserve the prior checkpoint and responsibility until a committed publication or exact readback confirms the transition; no retry or execution completion supplies independent verification.
|
||||
|
||||
Partial-write MRF replay uses Deep verification so a protected object's repair
|
||||
can discharge its obligation after verifying the payload. This adds full-object
|
||||
read work to those background attempts, including healthy replay targets.
|
||||
|
||||
Reference in new issue
Block a user