Files
mvdbeek 09541b6417 Avoid replaying workflows while tool inputs are unchanged
Galaxy 26.2 retries untracked workflow delays on every monitor iteration.
That is needed for conditions the scheduler cannot observe, but internal
tools also use this path while waiting for input data. Their readiness
exceptions lose the blocking input before reaching the scheduler. A
populated collection with running members or unresolved datatypes can
therefore trigger full workflow evaluation on every pass, rebuilding
modules and recovering outputs without making progress.

Input readiness checks now pass their blocking dependency through to the
scheduler. Each delayed step records one blocking dataset and is retried
when that dataset's effective state, datatype, or deletion flags change.
Collection population alone is insufficient because members may still be
running or waiting for datatype resolution. The readiness check respects
HDA state overrides, including paused inputs.

The blocker is taken from datasets already visited by the input checks
or selected with the existing nested-collection query builder. Subsequent
monitor passes query only the selected dataset columns, in batches of at
most 500 IDs, avoiding repeated collection scans and ORM object loading.
One blocker is sufficient to hold a step; the next evaluation discovers
any remaining waits. This can defer noticing a failure in another input
until the selected blocker changes, while independent workflow branches
retain their own dependencies.
2026-11-06 18:44:39 +01:00

203 lines
7.3 KiB
Python

from datetime import timedelta
from functools import partial
from typing import cast
import yaml
from sqlalchemy import (
delete,
insert,
)
from galaxy import model
from galaxy.app_unittest_utils import galaxy_mock
from galaxy.managers.tools import DynamicToolManager
from galaxy.managers.workflows import WorkflowsManager
from galaxy.structured_app import MinimalManagerApp
from galaxy.tools.parameters.workflow_utils import workflow_building_modes
from galaxy.util import now
from galaxy.util.bunch import Bunch
from galaxy.workflow.modules import module_factory
from galaxy.workflow.scheduling_manager import (
WorkflowRequestMonitor,
WorkflowSchedulingManager,
)
class MockTrans:
def __init__(self):
self.app = MockApp()
self.app.trs_proxy = Bunch()
self.security = self.app.security
self.sa_session = self.app.model.context
self.security = self.app.security
self.workflow_building_mode = workflow_building_modes.DISABLED
self._user = None
def save_workflow(self, workflow):
stored_workflow = model.StoredWorkflow()
stored_workflow.latest_workflow = workflow
workflow.stored_workflow = stored_workflow
stored_workflow.user = self.user
self.sa_session.add(stored_workflow)
self.sa_session.commit()
return stored_workflow
@property
def user(self):
if self._user is None:
self._user = model.User(email="testworkflows@bx.psu.edu", password="password")
return self._user
class MockApp(galaxy_mock.MockApp):
def __init__(self):
super().__init__()
self._toolbox = MockToolbox()
self.workflow_manager = WorkflowsManager(self, DynamicToolManager(self))
class MockDatatypesRegistry:
def __init__(self):
pass
def get_datatype_by_extension(self, ext):
return ext
@property
def datatypes_by_extension(self):
return {"fasta": object(), "fastqsanger": object(), "txt": object()}
class MockToolbox:
def __init__(self):
self.tools = {}
def get_tool(self, tool_id, tool_version=None, exact=False, tool_uuid=None, user=None):
# Real tool box returns None of missing tool also
return self.tools.get(tool_id, None)
def materialize_tool(self, tool, *, reason):
return tool
def yaml_to_model(has_dict, id_offset=100):
if isinstance(has_dict, str):
has_dict = yaml.safe_load(has_dict)
trans = MockTrans()
workflow = model.Workflow()
workflow.steps = []
for i, step in enumerate(has_dict.get("steps", [])):
workflow_step = model.WorkflowStep()
if "order_index" not in step:
step["order_index"] = i
if "id" not in step:
# Fixed Offset ids just to test against assuption order_index != id
step["id"] = id_offset
id_offset += 1
step_type = step.get("type", None)
assert step_type is not None
if step_type == "subworkflow":
subworkflow_dict = step["subworkflow"]
del step["subworkflow"]
subworkflow = yaml_to_model(subworkflow_dict, id_offset=id_offset)
step["subworkflow"] = subworkflow
id_offset += len(subworkflow.steps)
for key, value in step.items():
if key == "input_connections":
raise NotImplementedError()
if key == "inputs":
inputs = []
for input_name, input_def in value.items():
step_input = model.WorkflowStepInput(workflow_step)
step_input.name = input_name
connections = []
for conn_dict in input_def.get("connections", []):
conn = model.WorkflowStepConnection()
for conn_key, conn_value in conn_dict.items():
if conn_key == "@output_step":
target_step = workflow.steps[conn_value]
conn_value = target_step
conn_key = "output_step"
if conn_key == "@input_subworkflow_step":
conn_value = step["subworkflow"].step_by_index(conn_value)
conn_key = "input_subworkflow_step"
setattr(conn, conn_key, conn_value)
connections.append(conn)
step_input.connections = connections
inputs.append(step_input)
value = inputs
if key == "workflow_outputs":
value = [partial(_dict_to_workflow_output, workflow_step)(_) for _ in value]
if key == "collection_type":
key = "tool_inputs"
value = {"collection_type": value}
setattr(workflow_step, key, value)
if workflow_step.type != "tool":
module = module_factory.from_workflow_step(trans, workflow_step)
module.save_to_step(workflow_step)
workflow.steps.append(workflow_step)
return workflow
def _dict_to_workflow_output(workflow_step, as_dict):
output = model.WorkflowOutput(workflow_step)
for key, value in as_dict.items():
setattr(output, key, value)
return output
def expression_json_hda(path, content: str | None, seconds_since_updated: int = 0):
hda = model.HistoryDatasetAssociation(extension="expression.json", create_dataset=True, flush=False)
hda.id = 1
assert hda.dataset is not None
hda.dataset.state = model.Dataset.states.OK
if content is not None:
path.write_text(content)
hda.dataset.external_filename = str(path)
hda.update_time = now() - timedelta(seconds=seconds_since_updated)
return hda
class SchedulingHarness:
def __init__(self, **config):
self.app = galaxy_mock.MockApp(
config=galaxy_mock.MockAppConfig(
maximum_workflow_invocation_duration=0, history_local_serial_workflow_scheduling=False, **config
)
)
self.app.job_config["self_handler_tags"] = []
self.monitor = WorkflowRequestMonitor(
cast(MinimalManagerApp, self.app),
cast(WorkflowSchedulingManager, Bunch(default_handler_id="main", handler_assignment_methods=[])),
)
self.session = self.app.model.context
self.history = model.History()
invocation = model.WorkflowInvocation()
invocation.history = self.history
invocation.workflow = model.Workflow()
invocation.state = model.WorkflowInvocation.states.READY
self.session.add(invocation)
self.session.commit()
self.invocation_id = invocation.id
self.set_history_update_time(now() - timedelta(seconds=60))
def attempt(self, scheduler) -> int:
self.monitor._WorkflowRequestMonitor__attempt_schedule(self.invocation_id, scheduler)
return scheduler.scheduled
def set_history_update_time(self, update_time):
history_id = self.history.id
self.session.execute(delete(model.HistoryAudit).where(model.HistoryAudit.history_id == history_id))
self.session.execute(insert(model.HistoryAudit).values(history_id=history_id, update_time=update_time))
self.session.commit()
def persist(self, *objects):
self.session.add_all(objects)
self.session.commit()
return objects[0].id