mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-11-22 14:57:42 +08:00
Galaxy 26.2 retries untracked workflow delays on every monitor iteration. That is needed for conditions the scheduler cannot observe, but internal tools also use this path while waiting for input data. Their readiness exceptions lose the blocking input before reaching the scheduler. A populated collection with running members or unresolved datatypes can therefore trigger full workflow evaluation on every pass, rebuilding modules and recovering outputs without making progress. Input readiness checks now pass their blocking dependency through to the scheduler. Each delayed step records one blocking dataset and is retried when that dataset's effective state, datatype, or deletion flags change. Collection population alone is insufficient because members may still be running or waiting for datatype resolution. The readiness check respects HDA state overrides, including paused inputs. The blocker is taken from datasets already visited by the input checks or selected with the existing nested-collection query builder. Subsequent monitor passes query only the selected dataset columns, in batches of at most 500 IDs, avoiding repeated collection scans and ORM object loading. One blocker is sufficient to hold a step; the next evaluation discovers any remaining waits. This can defer noticing a failure in another input until the selected blocker changes, while independent workflow branches retain their own dependencies.
203 lines
7.3 KiB
Python
203 lines
7.3 KiB
Python
from datetime import timedelta
|
|
from functools import partial
|
|
from typing import cast
|
|
|
|
import yaml
|
|
from sqlalchemy import (
|
|
delete,
|
|
insert,
|
|
)
|
|
|
|
from galaxy import model
|
|
from galaxy.app_unittest_utils import galaxy_mock
|
|
from galaxy.managers.tools import DynamicToolManager
|
|
from galaxy.managers.workflows import WorkflowsManager
|
|
from galaxy.structured_app import MinimalManagerApp
|
|
from galaxy.tools.parameters.workflow_utils import workflow_building_modes
|
|
from galaxy.util import now
|
|
from galaxy.util.bunch import Bunch
|
|
from galaxy.workflow.modules import module_factory
|
|
from galaxy.workflow.scheduling_manager import (
|
|
WorkflowRequestMonitor,
|
|
WorkflowSchedulingManager,
|
|
)
|
|
|
|
|
|
class MockTrans:
|
|
def __init__(self):
|
|
self.app = MockApp()
|
|
self.app.trs_proxy = Bunch()
|
|
self.security = self.app.security
|
|
self.sa_session = self.app.model.context
|
|
self.security = self.app.security
|
|
self.workflow_building_mode = workflow_building_modes.DISABLED
|
|
self._user = None
|
|
|
|
def save_workflow(self, workflow):
|
|
stored_workflow = model.StoredWorkflow()
|
|
stored_workflow.latest_workflow = workflow
|
|
workflow.stored_workflow = stored_workflow
|
|
stored_workflow.user = self.user
|
|
self.sa_session.add(stored_workflow)
|
|
self.sa_session.commit()
|
|
return stored_workflow
|
|
|
|
@property
|
|
def user(self):
|
|
if self._user is None:
|
|
self._user = model.User(email="testworkflows@bx.psu.edu", password="password")
|
|
return self._user
|
|
|
|
|
|
class MockApp(galaxy_mock.MockApp):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self._toolbox = MockToolbox()
|
|
self.workflow_manager = WorkflowsManager(self, DynamicToolManager(self))
|
|
|
|
|
|
class MockDatatypesRegistry:
|
|
def __init__(self):
|
|
pass
|
|
|
|
def get_datatype_by_extension(self, ext):
|
|
return ext
|
|
|
|
@property
|
|
def datatypes_by_extension(self):
|
|
return {"fasta": object(), "fastqsanger": object(), "txt": object()}
|
|
|
|
|
|
class MockToolbox:
|
|
def __init__(self):
|
|
self.tools = {}
|
|
|
|
def get_tool(self, tool_id, tool_version=None, exact=False, tool_uuid=None, user=None):
|
|
# Real tool box returns None of missing tool also
|
|
return self.tools.get(tool_id, None)
|
|
|
|
def materialize_tool(self, tool, *, reason):
|
|
return tool
|
|
|
|
|
|
def yaml_to_model(has_dict, id_offset=100):
|
|
if isinstance(has_dict, str):
|
|
has_dict = yaml.safe_load(has_dict)
|
|
|
|
trans = MockTrans()
|
|
|
|
workflow = model.Workflow()
|
|
workflow.steps = []
|
|
for i, step in enumerate(has_dict.get("steps", [])):
|
|
workflow_step = model.WorkflowStep()
|
|
if "order_index" not in step:
|
|
step["order_index"] = i
|
|
if "id" not in step:
|
|
# Fixed Offset ids just to test against assuption order_index != id
|
|
step["id"] = id_offset
|
|
id_offset += 1
|
|
step_type = step.get("type", None)
|
|
assert step_type is not None
|
|
|
|
if step_type == "subworkflow":
|
|
subworkflow_dict = step["subworkflow"]
|
|
del step["subworkflow"]
|
|
subworkflow = yaml_to_model(subworkflow_dict, id_offset=id_offset)
|
|
step["subworkflow"] = subworkflow
|
|
id_offset += len(subworkflow.steps)
|
|
|
|
for key, value in step.items():
|
|
if key == "input_connections":
|
|
raise NotImplementedError()
|
|
if key == "inputs":
|
|
inputs = []
|
|
for input_name, input_def in value.items():
|
|
step_input = model.WorkflowStepInput(workflow_step)
|
|
step_input.name = input_name
|
|
connections = []
|
|
for conn_dict in input_def.get("connections", []):
|
|
conn = model.WorkflowStepConnection()
|
|
for conn_key, conn_value in conn_dict.items():
|
|
if conn_key == "@output_step":
|
|
target_step = workflow.steps[conn_value]
|
|
conn_value = target_step
|
|
conn_key = "output_step"
|
|
if conn_key == "@input_subworkflow_step":
|
|
conn_value = step["subworkflow"].step_by_index(conn_value)
|
|
conn_key = "input_subworkflow_step"
|
|
setattr(conn, conn_key, conn_value)
|
|
connections.append(conn)
|
|
step_input.connections = connections
|
|
inputs.append(step_input)
|
|
value = inputs
|
|
if key == "workflow_outputs":
|
|
value = [partial(_dict_to_workflow_output, workflow_step)(_) for _ in value]
|
|
if key == "collection_type":
|
|
key = "tool_inputs"
|
|
value = {"collection_type": value}
|
|
setattr(workflow_step, key, value)
|
|
if workflow_step.type != "tool":
|
|
module = module_factory.from_workflow_step(trans, workflow_step)
|
|
module.save_to_step(workflow_step)
|
|
workflow.steps.append(workflow_step)
|
|
|
|
return workflow
|
|
|
|
|
|
def _dict_to_workflow_output(workflow_step, as_dict):
|
|
output = model.WorkflowOutput(workflow_step)
|
|
for key, value in as_dict.items():
|
|
setattr(output, key, value)
|
|
return output
|
|
|
|
|
|
def expression_json_hda(path, content: str | None, seconds_since_updated: int = 0):
|
|
hda = model.HistoryDatasetAssociation(extension="expression.json", create_dataset=True, flush=False)
|
|
hda.id = 1
|
|
assert hda.dataset is not None
|
|
hda.dataset.state = model.Dataset.states.OK
|
|
if content is not None:
|
|
path.write_text(content)
|
|
hda.dataset.external_filename = str(path)
|
|
hda.update_time = now() - timedelta(seconds=seconds_since_updated)
|
|
return hda
|
|
|
|
|
|
class SchedulingHarness:
|
|
def __init__(self, **config):
|
|
self.app = galaxy_mock.MockApp(
|
|
config=galaxy_mock.MockAppConfig(
|
|
maximum_workflow_invocation_duration=0, history_local_serial_workflow_scheduling=False, **config
|
|
)
|
|
)
|
|
self.app.job_config["self_handler_tags"] = []
|
|
self.monitor = WorkflowRequestMonitor(
|
|
cast(MinimalManagerApp, self.app),
|
|
cast(WorkflowSchedulingManager, Bunch(default_handler_id="main", handler_assignment_methods=[])),
|
|
)
|
|
self.session = self.app.model.context
|
|
self.history = model.History()
|
|
invocation = model.WorkflowInvocation()
|
|
invocation.history = self.history
|
|
invocation.workflow = model.Workflow()
|
|
invocation.state = model.WorkflowInvocation.states.READY
|
|
self.session.add(invocation)
|
|
self.session.commit()
|
|
self.invocation_id = invocation.id
|
|
self.set_history_update_time(now() - timedelta(seconds=60))
|
|
|
|
def attempt(self, scheduler) -> int:
|
|
self.monitor._WorkflowRequestMonitor__attempt_schedule(self.invocation_id, scheduler)
|
|
return scheduler.scheduled
|
|
|
|
def set_history_update_time(self, update_time):
|
|
history_id = self.history.id
|
|
self.session.execute(delete(model.HistoryAudit).where(model.HistoryAudit.history_id == history_id))
|
|
self.session.execute(insert(model.HistoryAudit).values(history_id=history_id, update_time=update_time))
|
|
self.session.commit()
|
|
|
|
def persist(self, *objects):
|
|
self.session.add_all(objects)
|
|
self.session.commit()
|
|
return objects[0].id
|