Files
galaxy/test/unit/workflows/test_workflow_progress.py
John ChiltonandClaude Opus 5.5 f2ffabe3d7 Track connection HDA waits by readiness snapshot
A paused input on a require_ready connection was recorded as a legacy HDA
dependency, which counts paused as ready and replayed every pass.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-10 11:29:54 +01:00

430 lines
16 KiB
Python

from typing import cast
import pytest
from galaxy import model
from galaxy.model.dataset_collections.readiness import dataset_readiness
from galaxy.schema.invocation import FailureReason
from galaxy.util.unittest import TestCase
from galaxy.workflow import modules
from galaxy.workflow.run import (
ModuleInjector,
STEP_OUTPUT_DELAYED,
WorkflowProgress,
)
from galaxy.workflow.scheduling_dependencies import (
DependencyType,
SchedulingDependencies,
SchedulingDependency,
unpopulated_collection_dependencies,
)
from .workflow_support import (
expression_json_hda,
MockApp,
MockTrans,
yaml_to_model,
)
TEST_WORKFLOW_YAML = """
steps:
- type: "data_input"
tool_inputs: {"name": "input1"}
label: "input1"
- type: "data_input"
tool_inputs: {"name": "input2"}
- type: "tool"
tool_id: "cat1"
inputs:
"input1":
connections:
- "@output_step": 0
output_name: "output"
- type: "tool"
tool_id: "cat1"
inputs:
input1:
connections:
- "@output_step": 0
output_name: "output"
- type: "tool"
tool_id: "cat1"
inputs:
"input1":
connections:
- "@output_step": 2
output_name: "out_file1"
"""
TEST_SUBWORKFLOW_YAML = """
steps:
- type: "data_input"
tool_inputs: {"name": "outer_input"}
- type: "subworkflow"
subworkflow:
steps:
- type: "data_input"
tool_inputs: {"name": "inner_input"}
- type: "tool"
tool_id: "cat1"
inputs:
"input1":
connections:
- "@output_step": 0
output_name: "output"
inputs:
inner_input:
connections:
- "@output_step": 0
output_name: "output"
"@input_subworkflow_step": 0
"""
TEST_SUBWORKFLOW_PARAMETER_YAML = """
steps:
- type: "tool"
tool_id: "pick_value"
- type: "subworkflow"
subworkflow:
steps:
- type: "parameter_input"
label: "inner_text"
tool_inputs: {"parameter_type": "text", "optional": false}
inputs:
inner_text:
connections:
- "@output_step": 0
output_name: "text_param"
"@input_subworkflow_step": 0
"""
UNSCHEDULED_STEP = object()
class TestWorkflowProgress(TestCase):
def setUp(self):
self.app = MockApp()
self.inputs_by_step_id = {}
self.invocation = model.WorkflowInvocation()
self.progress = {}
def _setup_workflow(self, workflow_yaml):
workflow = yaml_to_model(workflow_yaml)
self.invocation.workflow = workflow
def _new_workflow_progress(self):
mock_injector: ModuleInjector = cast(ModuleInjector, MockModuleInjector(self.progress))
return WorkflowProgress(self.invocation, self.inputs_by_step_id, mock_injector, {})
def _set_previous_progress(self, outputs):
for i, (step_id, step_value) in enumerate(outputs):
if step_value is not UNSCHEDULED_STEP:
self.progress[step_id] = step_value
workflow_invocation_step = model.WorkflowInvocationStep()
workflow_invocation_step.workflow_step_id = step_id
workflow_invocation_step.state = "scheduled"
workflow_invocation_step.workflow_step = self._step(i)
assert step_id == self._step(i).id
workflow_invocation_step.workflow_invocation = self.invocation
workflow_invocation_step_state = model.WorkflowRequestStepState()
workflow_invocation_step_state.workflow_step_id = step_id
workflow_invocation_step_state.value = {"my_param": True}
self.invocation.step_states.append(workflow_invocation_step_state)
def _step(self, index):
return self.invocation.workflow.steps[index]
def _invocation_step(self, index):
if index < len(self.invocation.steps):
return self.invocation.steps[index]
else:
workflow_invocation_step = model.WorkflowInvocationStep()
workflow_invocation_step.workflow_step = self._step(index)
workflow_invocation_step.workflow_invocation = self.invocation
return workflow_invocation_step
def test_connect_data_input(self):
self._setup_workflow(TEST_WORKFLOW_YAML)
hda = model.HistoryDatasetAssociation()
self.inputs_by_step_id = {100: hda}
progress = self._new_workflow_progress()
progress.set_outputs_for_input(self._invocation_step(0))
conn = model.WorkflowStepConnection()
conn.output_name = "output"
conn.output_step = self._step(0)
assert progress.replacement_for_connection(conn) is hda
def test_replacement_for_tool_input(self):
self._setup_workflow(TEST_WORKFLOW_YAML)
hda = model.HistoryDatasetAssociation()
self.inputs_by_step_id = {100: hda}
progress = self._new_workflow_progress()
progress.set_outputs_for_input(self._invocation_step(0))
step_dict = {
"name": "input1",
"input_type": "dataset",
"multiple": False,
}
replacement = progress.replacement_for_input(None, self._step(2), step_dict)
assert replacement is hda
def test_connect_tool_output(self):
self._setup_workflow(TEST_WORKFLOW_YAML)
hda = model.HistoryDatasetAssociation()
progress = self._new_workflow_progress()
progress.set_step_outputs(self._invocation_step(2), {"out1": hda})
conn = model.WorkflowStepConnection()
conn.output_name = "out1"
conn.output_step = self._step(2)
assert progress.replacement_for_connection(conn) is hda
def test_remaining_steps_with_progress(self):
self._setup_workflow(TEST_WORKFLOW_YAML)
hda3 = model.HistoryDatasetAssociation()
self._set_previous_progress(
[
(100, {"output": model.HistoryDatasetAssociation()}),
(101, {"output": model.HistoryDatasetAssociation()}),
(102, {"out_file1": hda3}),
(103, {"out_file1": model.HistoryDatasetAssociation()}),
(104, UNSCHEDULED_STEP),
]
)
progress = self._new_workflow_progress()
steps = progress.remaining_steps()
assert len(steps) == 1, steps
step, invocation_step = steps[0]
assert step is self.invocation.workflow.steps[4]
step_dict = {
"name": "input1",
"input_type": "dataset",
"multiple": False,
}
replacement = progress.replacement_for_input(None, self._step(4), step_dict)
assert replacement is hda3
# TODO: Replace multiple true HDA with HDCA
# TODO: Test cancel on collection invalid
# TODO: Test delay on collection waiting for population
def _connection_to_step_2_output(self):
conn = model.WorkflowStepConnection()
conn.output_name = "out1"
conn.output_step = self._step(2)
return conn
def test_pending_dataset_delays_with_tracked_dependency(self):
self._assert_delays_with_readiness_dependency(model.Dataset.states.QUEUED, is_data=False)
def test_paused_dataset_delays_with_readiness_dependency_when_ready_required(self):
self._assert_delays_with_readiness_dependency(model.Dataset.states.PAUSED, require_ready=True)
def _assert_delays_with_readiness_dependency(self, state, **kwds):
self._setup_workflow(TEST_WORKFLOW_YAML)
dataset = model.Dataset()
dataset.state = state
hda = model.HistoryDatasetAssociation(dataset=dataset)
hda.id = 7
progress = self._new_workflow_progress()
progress.set_step_outputs(self._invocation_step(2), {"out1": hda})
with pytest.raises(modules.DelayedWorkflowEvaluation) as delayed:
progress.replacement_for_connection(self._connection_to_step_2_output(), **kwds)
progress.record_delay(delayed.value)
assert progress.scheduling_dependencies() == SchedulingDependencies(
tracked=frozenset({SchedulingDependency(DependencyType.HDA, 7, dataset_readiness(hda))}),
untracked=(),
more_work=False,
)
def test_nested_collection_depends_on_unpopulated_subcollections(self):
collection = model.DatasetCollection(collection_type="list:list")
populated_child = model.DatasetCollection(collection_type="list")
unpopulated_child = model.DatasetCollection(collection_type="list", populated=False)
for identifier, child in (("a", populated_child), ("b", unpopulated_child)):
model.DatasetCollectionElement(collection=collection, element=child, element_identifier=identifier)
session = self.app.model.session
session.add(collection)
session.commit()
assert unpopulated_collection_dependencies(collection) == [
SchedulingDependency(DependencyType.DATASET_COLLECTION, unpopulated_child.id)
]
def test_delay_inherited_from_delayed_step_is_not_untracked(self):
self._setup_workflow(TEST_WORKFLOW_YAML)
progress = self._new_workflow_progress()
progress.mark_step_outputs_delayed(self._step(2), why="waiting on something tracked elsewhere")
with pytest.raises(modules.DelayedWorkflowEvaluation) as delayed:
progress.replacement_for_connection(self._connection_to_step_2_output())
assert delayed.value.inherited
progress.record_delay(delayed.value)
assert progress.scheduling_dependencies() == SchedulingDependencies(
tracked=frozenset(), untracked=(), more_work=False
)
def test_delay_without_dependency_is_untracked(self):
self._setup_workflow(TEST_WORKFLOW_YAML)
progress = self._new_workflow_progress()
progress.record_delay(modules.DelayedWorkflowEvaluation(why="tool inputs are not ready"))
progress.record_delay(modules.DelayedWorkflowEvaluation())
assert progress.scheduling_dependencies().untracked == ("tool inputs are not ready", "no reason given")
def test_subworkflow_progress(self):
self._setup_workflow(TEST_SUBWORKFLOW_YAML)
hda = model.HistoryDatasetAssociation()
self._set_previous_progress(
[
(100, {"output": hda}),
(101, UNSCHEDULED_STEP),
]
)
subworkflow_invocation = self.invocation.create_subworkflow_invocation_for_step(
self.invocation.workflow.step_by_index(1)
)
session = self.app.model.session
session.add(self.invocation)
session.add(subworkflow_invocation)
session.commit()
progress = self._new_workflow_progress()
remaining_steps = progress.remaining_steps()
subworkflow_step, subworkflow_invocation_step = remaining_steps[0]
subworkflow_progress = progress.subworkflow_progress(subworkflow_invocation, subworkflow_step, {})
subworkflow = subworkflow_step.subworkflow
assert subworkflow_progress.workflow_invocation == subworkflow_invocation
assert subworkflow_progress.workflow_invocation.workflow == subworkflow
subworkflow_input_step = subworkflow.step_by_index(0)
subworkflow_invocation_step = model.WorkflowInvocationStep()
subworkflow_invocation_step.workflow_step_id = subworkflow_input_step.id
subworkflow_invocation_step.state = "new"
subworkflow_invocation_step.workflow_step = subworkflow_input_step
subworkflow_invocation_step.workflow_invocation = subworkflow_invocation
subworkflow_progress.set_outputs_for_input(subworkflow_invocation_step)
subworkflow_cat_step = subworkflow.step_by_index(1)
step_dict = {
"name": "input1",
"input_type": "dataset",
"multiple": False,
}
assert hda is subworkflow_progress.replacement_for_input(
None,
subworkflow_cat_step,
step_dict,
)
def test_subworkflow_unreadable_expression_json_parameter_delays_subworkflow(self, tmp_path):
hda = expression_json_hda(tmp_path / "expression.json", None)
subworkflow_invocation, subworkflow_step = self._setup_subworkflow_parameter(hda)
with pytest.raises(modules.DelayedWorkflowEvaluation) as exc_info:
self._recovered_workflow_progress().subworkflow_progress(subworkflow_invocation, subworkflow_step, {})
assert "could not be read" in exc_info.value.why
def _recovered_workflow_progress(self):
progress = self._new_workflow_progress()
progress.remaining_steps()
return progress
def _setup_subworkflow_parameter(self, hda):
self._setup_workflow(TEST_SUBWORKFLOW_PARAMETER_YAML)
self._set_previous_progress(
[
(100, {"text_param": hda}),
(101, UNSCHEDULED_STEP),
]
)
subworkflow_step = self.invocation.workflow.step_by_index(1)
subworkflow_invocation = self.invocation.create_subworkflow_invocation_for_step(subworkflow_step)
session = self.app.model.session
session.add(self.invocation)
session.add(subworkflow_invocation)
session.commit()
return subworkflow_invocation, subworkflow_step
@pytest.mark.parametrize(
"connection_state,missing_names",
[
("disconnected", "'1', 'missing_input'"),
("ready", "'missing_input'"),
("pending", "'missing_input'"),
],
)
def test_disconnected_subworkflow_runtime_failure(connection_state, missing_names):
# Persist a run directly: ordinary submission now rejects this saved graph.
app = MockApp()
workflow = yaml_to_model(TEST_SUBWORKFLOW_YAML)
subworkflow_step = workflow.step_by_index(1)
missing_input = model.WorkflowStep()
missing_input.id = 104
missing_input.order_index = 2
missing_input.type = "data_input"
missing_input.tool_inputs = {"name": "missing_input"}
subworkflow_step.subworkflow.steps.append(missing_input)
if connection_state == "disconnected":
subworkflow_step.inputs = []
invocation = model.WorkflowInvocation()
invocation.workflow = workflow
subworkflow_invocation = invocation.create_subworkflow_invocation_for_step(subworkflow_step)
session = app.model.session
session.add(invocation)
session.add(subworkflow_invocation)
session.commit()
injector = cast(ModuleInjector, MockModuleInjector({}))
progress = WorkflowProgress(invocation, {}, injector, {})
if connection_state == "ready":
progress.outputs[workflow.step_by_index(0).id] = {"output": model.HistoryDatasetAssociation()}
elif connection_state == "pending":
progress.outputs[workflow.step_by_index(0).id] = STEP_OUTPUT_DELAYED
with pytest.raises(modules.FailWorkflowEvaluation) as exc:
progress.subworkflow_progress(subworkflow_invocation, subworkflow_step, {})
assert exc.value.why.model_dump(exclude_none=True) == {
"reason": FailureReason.unexpected_failure,
"workflow_step_id": subworkflow_step.id,
"details": f"Subworkflow has disconnected required input {missing_names}",
}
class MockModuleInjector:
def __init__(self, progress):
self.progress = progress
def inject(self, step, step_args=None, steps=None, **kwargs):
step.module = MockModule(self.progress)
def inject_all(self, workflow, param_map=None, ignore_tool_missing_exception=True, **kwargs):
param_map = param_map or {}
for step in workflow.steps:
step_args = param_map.get(step.id, {})
self.inject(step, step_args=step_args)
def compute_runtime_state(self, step, step_args=None):
pass
class MockModule:
def __init__(self, progress):
self.progress = progress
self.trans = MockTrans()
def decode_runtime_state(self, step, runtime_state):
return True
def recover_mapping(self, invocation_step, progress):
step_id = invocation_step.workflow_step.id
if step_id in self.progress:
progress.set_step_outputs(invocation_step, self.progress[step_id])