Files

1207 lines
46 KiB
Python

import json
from typing import (
Any,
NamedTuple,
)
from unittest import mock
import pytest
from sqlalchemy.exc import IntegrityError
from galaxy import model
from galaxy.exceptions import (
InternalServerError,
RequestParameterInvalidException,
)
from galaxy.managers.workflows import WorkflowContentsManager
from galaxy.schema.invocation import FailureReason
from galaxy.tool_util.parameters import input_models_for_tool_source
from galaxy.tool_util.parser.output_objects import ToolOutput
from galaxy.tool_util.unittest_utils.parameters import parameter_tool_source
from galaxy.tools.parameters import populate_state
from galaxy.tools.parameters.basic import ParameterValueError
from galaxy.tools.parameters.meta import to_decoded_json
from galaxy.tools.parameters.workflow_utils import (
ConnectedValue,
NO_REPLACEMENT,
RuntimeValue,
workflow_building_modes,
)
from galaxy.util import (
bunch,
)
from galaxy.workflow import modules
from galaxy.workflow.workflow_parameter_input_definitions import split_numeric_bounds
from .workflow_support import (
expression_json_hda,
MockTrans,
yaml_to_model,
)
def test_input_has_no_errors():
trans = MockTrans()
input_step_module = modules.module_factory.from_dict(trans, {"type": "data_input"})
assert not input_step_module.get_errors()
def test_valid_new_tool_has_no_errors():
trans = MockTrans()
mock_tool = __mock_tool()
trans.app.toolbox.tools["cat1"] = mock_tool
tool_module = modules.module_factory.from_dict(trans, {"type": "tool", "tool_id": "cat1"})
assert not tool_module.get_errors()
def test_data_input_default_state():
trans = MockTrans()
module = modules.module_factory.from_dict(trans, {"type": "data_input", "label": "Input Dataset"})
__assert_has_runtime_input(module, label="Input Dataset")
def test_data_input_modified_state():
module = __from_state({"type": "data_input", "label": "Cool Input"})
__assert_has_runtime_input(module, label="Cool Input")
def test_data_input_step_modified_state():
module = __from_step(type="data_input", label="Cool Input")
__assert_has_runtime_input(module, label="Cool Input")
def test_data_input_compute_runtime_state_default():
module = __from_step(type="data_input")
state, errors = module.compute_runtime_state(module.trans, module.test_step)
assert not errors
assert "input" in state.inputs
assert state.inputs["input"] is NO_REPLACEMENT
def test_data_input_compute_runtime_state_args():
module = __from_step(type="data_input")
tool_state = module.get_state()
hda = model.HistoryDatasetAssociation()
with mock.patch("galaxy.workflow.modules.check_param") as check_method:
check_method.return_value = (hda, None)
state, errors = module.compute_runtime_state(
module.trans, module.test_step, {"input": 4, "tool_state": tool_state}
)
assert not errors
assert "input" in state.inputs
assert state.inputs["input"] is hda
def test_data_input_connections():
module = __from_step(type="data_input")
assert len(module.get_data_inputs()) == 0
outputs = module.get_data_outputs()
assert len(outputs) == 1
output = outputs[0]
assert output["name"] == "output"
assert output["extensions"] == ["input"]
def test_data_collection_input_default_state():
trans = MockTrans()
module = modules.module_factory.from_dict(
trans, {"type": "data_collection_input", "label": "Input Dataset Collection"}
)
__assert_has_runtime_input(module, label="Input Dataset Collection", collection_type="list")
def test_data_input_collection_modified_state():
module = __from_state(
{
"type": "data_collection_input",
"label": "Cool Input Collection",
"tool_state": json.dumps({"collection_type": "list:paired"}),
}
)
__assert_has_runtime_input(module, label="Cool Input Collection", collection_type="list:paired")
def test_data_input_collection_step_modified_state():
module = __from_step(
type="data_collection_input",
label="Cool Input Collection",
tool_inputs={
"collection_type": "list:paired",
},
)
__assert_has_runtime_input(module, label="Cool Input Collection", collection_type="list:paired")
def test_data_collection_input_connections():
module = __from_step(type="data_collection_input", tool_inputs={"collection_type": "list:paired"})
assert len(module.get_data_inputs()) == 0
outputs = module.get_data_outputs()
assert len(outputs) == 1
output = outputs[0]
assert output["name"] == "output"
assert output["extensions"] == ["input"]
assert output["collection_type"] == "list:paired"
def test_parameter_input_without_bounds_is_unbounded():
parameter, trans = __numeric_runtime_parameter("integer", None, None)
parameter.validate(-1000, trans)
field = parameter.to_dict(trans)
assert (field["min"], field["max"]) == (None, None)
NUMERIC_RANGE_CASES = [
("integer", 1, 5, 7, "Value ('7') must fulfill (1 <= value <= 5)"),
("integer", 0, None, -1, "Value ('-1') must fulfill (0 <= value <= +infinity)"),
("integer", None, -2, -1, "Value ('-1') must fulfill (-infinity <= value <= -2)"),
("float", 0.5, 2.5, 3.0, "Value ('3.0') must fulfill (0.5 <= value <= 2.5)"),
]
@pytest.mark.parametrize("parameter_type,min,max,invalid_value,message", NUMERIC_RANGE_CASES)
def test_parameter_input_range_is_validated_once(parameter_type, min, max, invalid_value, message):
parameter, trans = __numeric_runtime_parameter(parameter_type, min, max)
assert len(parameter.validators) == 1
with pytest.raises(ValueError) as exc_info:
parameter.validate(invalid_value, trans)
assert str(exc_info.value) == f"Parameter 'input': {message}"
@pytest.mark.parametrize(
"parameter_type,min,max,expected",
[(parameter_type, min, max, (min, max)) for parameter_type, min, max, *_ in NUMERIC_RANGE_CASES]
+ [("float", 1, 2, (1.0, 2.0))],
)
def test_parameter_input_range_reaches_run_form(parameter_type, min, max, expected):
parameter, trans = __numeric_runtime_parameter(parameter_type, min, max)
field = parameter.to_dict(trans)
assert (field["min"], field["max"]) == expected
@pytest.mark.parametrize(
"min,max,default,field_bounds",
[(1, 5, 3, (1, 5)), (1, 5, 7, (1, 5)), (5, 1, 3, (None, None))],
)
def test_parameter_input_editor_state_with_bounds(min, max, default, field_bounds):
# Building the editor form keeps saved state as-is, even for an out-of-range default or inverted bounds.
module = __numeric_editor_module("integer", min, max, default=default)
field = __editor_default_parameter(module, "integer").to_dict(module.trans)
assert (field["min"], field["max"]) == field_bounds
assert module.get_export_state()["default"] == default
assert module.get_export_state()["validators"] == [{"min": min, "max": max, "negate": False, "type": "in_range"}]
@pytest.mark.parametrize("parameter_type,min,max", [("integer", 1, 5), ("integer", 0, None), ("float", None, 2.5)])
def test_parameter_input_editor_default_has_bounds(parameter_type, min, max):
module = __numeric_editor_module(parameter_type, min, max)
field = __editor_default_parameter(module, parameter_type).to_dict(module.trans)
assert (field["min"], field["max"]) == (min, max)
other_type = "float" if parameter_type == "integer" else "integer"
other_field = __editor_default_parameter(module, other_type).to_dict(module.trans)
assert (other_field["min"], other_field["max"]) == (None, None)
def test_parameter_input_editor_default_follows_edited_bounds():
# Mirrors the editor's build_module: form values arrive without the previous tool state.
trans = MockTrans()
trans.workflow_building_mode = workflow_building_modes.ENABLED
module = modules.module_factory.from_dict(trans, {"type": "parameter_input"}, from_tool_form=True)
incoming = {
"parameter_definition|parameter_type": "integer",
"parameter_definition|optional|optional": "false",
"parameter_definition|optional|specify_default|specify_default": "true",
"parameter_definition|optional|specify_default|default": "3",
"parameter_definition|min": "2",
"parameter_definition|max": "4",
"parameter_definition|restrictions|how": "none",
}
state: dict[str, Any] = {}
errors: dict[str, Any] = {}
populate_state(trans, module.get_inputs(), incoming, state, errors=errors, check=True)
module.recover_state(state, from_tool_form=True)
assert not errors
assert module.get_export_state()["default"] == 3
field = __editor_default_parameter(module, "integer").to_dict(trans)
assert (field["min"], field["max"]) == (2, 4)
def test_parameter_input_inverted_bounds():
module = __numeric_editor_module("integer", 5, 1)
field = __editor_default_parameter(module, "integer").to_dict(module.trans)
assert (field["min"], field["max"]) == (None, None)
with pytest.raises(RequestParameterInvalidException, match="Workflow input 'Count' has a minimum"):
module.get_runtime_inputs(__step())
assert module.get_errors() == "Workflow input 'Count' has a minimum (5) greater than its maximum (1)."
def test_parameter_input_cleared_bounds_are_unbounded():
# Both editor fields cleared still saves an in_range validator, with no ends set.
module = __numeric_editor_module("integer", None, None)
field = __editor_default_parameter(module, "integer").to_dict(module.trans)
assert (field["min"], field["max"]) == (None, None)
assert module.get_errors() is None
@pytest.mark.parametrize(
"parameter_type,min,max,expected_bounds",
[
("integer", None, None, None),
("integer", 0, None, {"min": 0, "max": None}),
("float", None, 2.5, {"min": None, "max": 2.5}),
("text", 1, 5, None),
],
)
def test_split_numeric_bounds(parameter_type, min, max, expected_bounds):
range_validator = {"type": "in_range", "min": min, "max": max, "negate": False}
regex_validator = {"type": "regex", "expression": ".*", "negate": False}
bounds, validators = split_numeric_bounds(
{"parameter_type": parameter_type, "validators": [range_validator, regex_validator]}
)
assert bounds == expected_bounds
expected_validators = [range_validator, regex_validator] if parameter_type == "text" else [regex_validator]
assert validators == expected_validators
@pytest.mark.parametrize(
"parameter_type,range_validator",
[
("integer", {"type": "in_range", "min": 1, "max": 5, "negate": True}),
("integer", {"type": "in_range", "min": 1, "max": 5, "exclude_min": True}),
("integer", {"type": "in_range", "min": 1, "max": 5, "message": "1-5 only"}),
("integer", {"type": "in_range", "min": 1.5, "max": 5}),
],
)
def test_split_numeric_bounds_keeps_ranges_bounds_cannot_express(parameter_type, range_validator):
bounds, validators = split_numeric_bounds({"parameter_type": parameter_type, "validators": [range_validator]})
assert bounds is None
assert validators == [range_validator]
def test_split_numeric_bounds_separates_inverted_bounds():
bounds, validators = split_numeric_bounds(
{"parameter_type": "integer", "validators": [{"type": "in_range", "min": 5, "max": 1, "negate": False}]}
)
assert bounds == {"min": 5, "max": 1}
assert validators == []
def test_parameter_input_ordered_bounds_have_no_errors():
assert __numeric_editor_module("integer", 1, 5).get_errors() is None
assert __numeric_editor_module("float", 2.5, 2.5).get_errors() is None
@pytest.mark.parametrize("form_value,negate", [("true", True), ("false", False)])
def test_text_input_regex_validator_negate_from_tool_form(form_value, negate):
trans = MockTrans()
trans.workflow_building_mode = workflow_building_modes.ENABLED
module = modules.module_factory.from_dict(trans, {"type": "parameter_input"}, from_tool_form=True)
incoming = {
"parameter_definition|parameter_type": "text",
"parameter_definition|optional|optional": "false",
"parameter_definition|validators_0|regex_match": "cow",
"parameter_definition|validators_0|regex_doc": "no cows",
"parameter_definition|validators_0|regex_negate": form_value,
}
module_state = {}
errors = {}
populate_state(trans, module.get_inputs(), incoming, module_state, errors=errors, check=True)
assert not errors
module.recover_state(module_state, from_tool_form=True)
step = model.WorkflowStep()
module.save_to_step(step)
assert step.tool_inputs["validators"] == [
{"type": "regex", "expression": "cow", "negate": negate, "message": "no cows"}
]
def test_cannot_create_tool_modules_for_missing_tools():
trans = MockTrans()
module = modules.module_factory.from_dict(trans, {"type": "tool", "tool_id": "cat1"})
assert not module.tool
def test_view_only_tool_module_cannot_execute():
trans = MockTrans()
trans.app.toolbox.tools["cat1"] = __mock_tool(id="cat1", version="1.0")
step = __step(type="tool", tool_id="cat1", tool_version="1.0", config=None)
module = modules.module_factory.from_workflow_step(trans, step, view_only=True)
assert module.tool
with pytest.raises(InternalServerError):
module.execute(trans, mock.Mock(), mock.Mock())
def test_tool_version_latest_resolves_toolshed_guid():
# Toolshed GUIDs embed the version as the last segment. When tool_version="latest"
# is requested (as the WF editor does on insert), from_dict should strip the version
# from the GUID and resolve to the latest installed version via the versionless key.
trans = MockTrans()
old_guid = "toolshed.g2.bx.psu.edu/repos/devteam/fastqc/fastqc/0.68+galaxy1"
versionless_guid = "toolshed.g2.bx.psu.edu/repos/devteam/fastqc/fastqc"
latest_tool = __mock_tool(
id="toolshed.g2.bx.psu.edu/repos/devteam/fastqc/fastqc/0.74+galaxy1", version="0.74+galaxy1"
)
trans.app.toolbox.tools[versionless_guid] = latest_tool
module = modules.module_factory.from_dict(trans, {"type": "tool", "content_id": old_guid, "tool_version": "latest"})
assert module.tool is not None
assert module.tool.version == "0.74+galaxy1"
def test_tool_version_latest_resolves_builtin_tool():
# Built-in tool IDs have no version segment; remove_version_from_guid returns None
# so the ID is unchanged. tool_version="latest" should still resolve correctly.
trans = MockTrans()
latest_tool = __mock_tool(id="cat1", version="2.0")
trans.app.toolbox.tools["cat1"] = latest_tool
module = modules.module_factory.from_dict(trans, {"type": "tool", "content_id": "cat1", "tool_version": "latest"})
assert module.tool is not None
assert module.tool.version == "2.0"
def test_updated_tool_version():
trans = MockTrans()
mock_tool = __mock_tool(id="cat1", version="0.9")
trans.app.toolbox.tools["cat1"] = mock_tool
module = __from_step(
trans=trans,
type="tool",
tool_id="cat1",
tool_version="0.7",
config=None,
)
# Make sure there is a warnin with tool id, old version,
# and new version.
for val in "cat1", "0.7", "0.9":
assert val in module.version_changes[0]
def test_tool_version_same():
trans = MockTrans()
mock_tool = __mock_tool(id="cat1", version="1.0")
trans.app.toolbox.tools["cat1"] = mock_tool
module = __from_step(
trans=trans,
type="tool",
tool_id="cat1",
tool_version="1.0",
config=None,
)
assert not module.version_changes
TEST_WORKFLOW_YAML = """
steps:
- type: "data_input"
label: "input1"
- type: "data_collection_input"
label: "input2"
- type: "tool"
tool_id: "cat1"
inputs:
input1:
connections:
- "@output_step": 0
output_name: "output"
- type: "tool"
tool_id: "cat1"
inputs:
input1:
connections:
- "@output_step": 0
output_name: "output"
workflow_outputs:
- output_name: "out_file1"
label: "out1"
- type: "tool"
tool_id: "cat1"
inputs:
input1:
connections:
- "@output_step": 2
output_name: "out_file1"
workflow_outputs:
- output_name: "out_file1"
"""
COLLECTION_TYPE_WORKFLOW_YAML = """
steps:
- type: "data_collection_input"
label: "input1"
collection_type: "list:list"
- type: "tool"
tool_id: "cat1"
inputs:
input1:
connections:
- "@output_step": 0
output_name: "output"
"""
def test_subworkflow_new_inputs():
subworkflow_module = __new_subworkflow_module()
inputs = subworkflow_module.get_data_inputs()
assert len(inputs) == 2, len(inputs)
input1, input2 = inputs
assert input1["input_type"] == "dataset"
assert input1["name"] == "input1"
assert input2["input_type"] == "dataset_collection"
assert input2["name"] == "input2", input2["name"]
def test_subworkflow_new_inputs_collection_type():
subworkflow_module = __new_subworkflow_module(COLLECTION_TYPE_WORKFLOW_YAML)
inputs = subworkflow_module.get_data_inputs()
assert inputs[0]["collection_type"] == "list:list"
MULTIPLE_PARAMETER_WORKFLOW_YAML = """
steps:
- type: "parameter_input"
label: "single"
tool_inputs: {"parameter_type": "integer", "optional": false}
- type: "parameter_input"
label: "multiple"
tool_inputs: {"parameter_type": "integer", "optional": false, "multiple": true}
- type: "parameter_input"
label: "multiple_text"
tool_inputs: {"parameter_type": "text", "optional": false, "multiple": true}
"""
def test_subworkflow_new_inputs_parameter_multiple():
subworkflow_module = __new_subworkflow_module(MULTIPLE_PARAMETER_WORKFLOW_YAML)
inputs = {i["name"]: i for i in subworkflow_module.get_all_inputs()}
assert inputs["single"]["multiple"] is False
assert inputs["multiple"]["multiple"] is True
assert inputs["multiple_text"]["multiple"] is True
def test_parameter_input_multiple_integer_list_default_round_trip():
module, errors = __populate_parameter_from_tool_form(multiple=True, default=[1, 2])
assert not errors, errors
step = model.WorkflowStep()
module.save_to_step(step)
reloaded = modules.module_factory.from_workflow_step(MockTrans(), step)
assert reloaded.get_export_state()["default"] == [1, 2]
assert reloaded.get_runtime_inputs(mock.MagicMock())["input"].get_initial_value(None, {}) == [1, 2]
def test_parameter_input_multiple_text_list_default():
module, errors = __populate_parameter_from_tool_form(multiple=True, default=["a,b", "c"], parameter_type="text")
assert not errors, errors
default_fields = [
field
for field in __form_fields(module.get_config_form()["inputs"])
if field.get("label") == "Default Value" and field["type"] == "text"
]
assert all(field["multiple"] for field in default_fields)
assert ["a,b", "c"] in [field["value"] for field in default_fields]
step = model.WorkflowStep()
module.save_to_step(step)
reloaded = modules.module_factory.from_workflow_step(MockTrans(), step)
assert reloaded.get_runtime_inputs(mock.MagicMock())["input"].get_initial_value(None, {}) == ["a,b", "c"]
@pytest.mark.parametrize("default", ["1,2", "1\n2", ["1,2"]])
def test_parameter_input_multiple_integer_rejects_separated_string(default):
_, errors = __populate_parameter_from_tool_form(multiple=True, default=default)
assert "an integer is required" in str(errors["parameter_definition|optional|specify_default|default"])
def test_parameter_input_multiple_integer_cleared_default():
_, errors = __populate_parameter_from_tool_form(multiple=True, default=None)
assert "an integer or workflow parameter is required" in str(
errors["parameter_definition|optional|specify_default|default"]
)
def test_parameter_input_list_default_after_disabling_multiple():
_, errors = __populate_parameter_from_tool_form(multiple=False, default=[1, 2])
assert "an integer or workflow parameter is required" in str(
errors["parameter_definition|optional|specify_default|default"]
)
step = model.WorkflowStep()
step.type = "parameter_input"
step.tool_inputs = {"parameter_type": "integer", "optional": False, "multiple": False, "default": [1, 2]}
module = modules.module_factory.from_workflow_step(MockTrans(), step)
with pytest.raises(ParameterValueError, match="the attribute 'value' must be an integer"):
module.get_runtime_inputs(mock.MagicMock())
def test_parameter_input_text_list_default_after_disabling_multiple():
_, errors = __populate_parameter_from_tool_form(multiple=False, default=["a", "b"], parameter_type="text")
assert "a single value is required" in str(errors["parameter_definition|optional|specify_default|default"])
def test_subworkflow_new_outputs():
subworkflow_module = __new_subworkflow_module()
outputs = subworkflow_module.get_data_outputs()
assert len(outputs) == 2, len(outputs)
output1, output2 = outputs
assert output1["name"] == "out1"
assert output1["extensions"] == ["input"]
assert output2["name"] == "4:out_file1", output2["name"]
def test_to_cwl():
hda = model.HistoryDatasetAssociation(create_dataset=True, flush=False)
hda.dataset.state = model.Dataset.states.OK
hdas = [hda]
hda_references = []
result = modules.to_cwl(hdas, hda_references, model.WorkflowStep())
assert isinstance(result, list)
assert len(result) == 1
assert result[0]["class"] == "File"
assert hda_references == hdas
def test_to_cwl_purged_dataset():
hda = model.HistoryDatasetAssociation(create_dataset=True, flush=False)
hda.id = 1
hda.dataset.state = model.Dataset.states.OK
hda.dataset.purged = True
step = model.WorkflowStep()
step.id = 1
with pytest.raises(modules.FailWorkflowEvaluation):
modules.to_cwl(hda, [], step)
def test_to_cwl_nested_collection():
hda = model.HistoryDatasetAssociation(name="reads.fq", create_dataset=True, flush=False)
hda.dataset.state = model.Dataset.states.OK
dc_inner = model.DatasetCollection(collection_type="list")
model.DatasetCollectionElement(collection=dc_inner, element_identifier="inner", element=hda)
dc_outer = model.DatasetCollection(collection_type="list:list")
model.DatasetCollectionElement(collection=dc_outer, element_identifier="outer", element=dc_inner)
hdca = model.HistoryDatasetCollectionAssociation(name="the collection", collection=dc_outer)
result = modules.to_cwl(hdca, [], model.WorkflowStep())
assert result["outer"][0]["class"] == "File"
assert result["outer"][0]["basename"] == "reads.fq"
assert result["outer"][0]["element_identifier"] == "inner"
def test_to_cwl_dataset_collection_element():
hda = model.HistoryDatasetAssociation(name="reads.fq", create_dataset=True, flush=False)
hda.dataset.state = model.Dataset.states.OK
dc_inner = model.DatasetCollection(collection_type="list")
model.DatasetCollectionElement(collection=dc_inner, element_identifier="inner", element=hda)
dc_outer = model.DatasetCollection(collection_type="list:list")
dce_outer = model.DatasetCollectionElement(collection=dc_outer, element_identifier="outer", element=dc_inner)
result = modules.to_cwl(dce_outer, [], model.WorkflowStep())
assert result[0]["class"] == "File"
assert result[0]["basename"] == "reads.fq"
assert result[0]["element_identifier"] == "inner"
def test_to_cwl_record_collection():
dc = model.DatasetCollection(collection_type="paired")
for identifier, name in (("forward", "r1.fq"), ("reverse", "r2.fq")):
hda = model.HistoryDatasetAssociation(name=name, create_dataset=True, flush=False)
hda.dataset.state = model.Dataset.states.OK
model.DatasetCollectionElement(collection=dc, element_identifier=identifier, element=hda)
hdca = model.HistoryDatasetCollectionAssociation(name="the pair", collection=dc)
result = modules.to_cwl(hdca, [], model.WorkflowStep())
assert {k: (v["basename"], v["element_identifier"]) for k, v in result.items()} == {
"forward": ("r1.fq", "forward"),
"reverse": ("r2.fq", "reverse"),
}
def _workflow_step():
step = model.WorkflowStep()
step.id = 1
return step
def test_to_cwl_expression_json(tmp_path):
hda = expression_json_hda(tmp_path / "expression.json", '"abs(c3)>0.5"')
assert modules.to_cwl(hda, [], _workflow_step()) == "abs(c3)>0.5"
def test_to_cwl_expression_json_empty_file_recently_updated_delays(tmp_path):
hda = expression_json_hda(tmp_path / "expression.json", "")
with pytest.raises(modules.DelayedWorkflowEvaluation) as exc_info:
modules.to_cwl(hda, [], _workflow_step())
assert "could not be read" in exc_info.value.why
def test_to_cwl_expression_json_malformed_fails_after_grace_period(tmp_path):
hda = expression_json_hda(
tmp_path / "expression.json",
"{not json",
seconds_since_updated=modules.EXPRESSION_JSON_GRACE_PERIOD_SECONDS + 1,
)
step = _workflow_step()
with pytest.raises(modules.FailWorkflowEvaluation) as exc_info:
modules.to_cwl(hda, [], step)
why = exc_info.value.why
assert why.reason == FailureReason.unexpected_failure
assert why.workflow_step_id == step.id
assert "dataset 1" in why.details
def test_to_cwl_expression_json_missing_file_fails_after_grace_period(tmp_path):
hda = expression_json_hda(
tmp_path / "expression.json", None, seconds_since_updated=modules.EXPRESSION_JSON_GRACE_PERIOD_SECONDS + 1
)
with pytest.raises(modules.FailWorkflowEvaluation) as exc_info:
modules.to_cwl(hda, [], _workflow_step())
why = exc_info.value.why
assert why.reason == FailureReason.unexpected_failure
# Invocation details must not expose the dataset path.
assert str(tmp_path) not in why.details
def test_read_expression_json_without_update_time_fails_immediately(tmp_path):
hda = expression_json_hda(tmp_path / "expression.json", "")
hda.update_time = None
with pytest.raises(modules.FailWorkflowEvaluation) as exc_info:
modules.read_expression_json(hda, _workflow_step())
assert exc_info.value.why.reason == FailureReason.unexpected_failure
def test_read_expression_json_does_not_swallow_other_os_errors(tmp_path):
a_directory = tmp_path / "expression.json"
a_directory.mkdir()
hda = expression_json_hda(a_directory, None)
with pytest.raises(IsADirectoryError):
modules.read_expression_json(hda, _workflow_step())
def test_read_expression_json_without_step_reraises_read_error(tmp_path):
hda = expression_json_hda(tmp_path / "expression.json", "")
with pytest.raises(json.JSONDecodeError):
modules.read_expression_json(hda)
def test_replace_expression_json_dataset(tmp_path):
hda = expression_json_hda(tmp_path / "expression.json", "0.5")
assert modules.replace_expression_json_dataset(hda, _workflow_step()) == 0.5
def test_replace_expression_json_dataset_collection_element(tmp_path):
hda = expression_json_hda(tmp_path / "expression.json", "true")
collection = model.DatasetCollection(collection_type="list")
element = model.DatasetCollectionElement(collection=collection, element_identifier="true", element=hda)
assert modules.replace_expression_json_dataset(element, _workflow_step()) is True
def test_replace_expression_json_dataset_leaves_other_replacements_unchanged():
hda = model.HistoryDatasetAssociation(extension="txt", create_dataset=True, flush=False)
assert modules.replace_expression_json_dataset(hda, _workflow_step()) is hda
assert modules.replace_expression_json_dataset("a string", _workflow_step()) == "a string"
class MapOverTestCase(NamedTuple):
data_input: str
step_input_def: str | list[str]
step_output_def: str
expected_collection_type: str | None
steps: dict[int, Any]
def _construct_steps_for_map_over() -> list[MapOverTestCase]:
test_case = MapOverTestCase
# these are the cartesian product of
# data_input = ['dataset', 'list', 'list:pair', 'list:list']
# step_input_definition = ['dataset', 'dataset_multiple', 'list', ['list', 'pair']]
# step_output_definition = ['dataset', 'list', 'list:list']
# list(itertools.product(data_input, step_input_definition, step_output_definition, [None])),
# with the last item filled in manually
test_case_args: list[tuple[str, str | list[str], str, str | None]] = [
("dataset", "dataset", "dataset", None),
("dataset", "dataset", "list", "list"),
("dataset", "dataset", "list:list", "list:list"),
("dataset", "dataset_multiple", "dataset", None),
("dataset", "dataset_multiple", "list", "list"),
("dataset", "dataset_multiple", "list:list", "list:list"),
# Can't feed a dataset into a list or pair input
# ('dataset', 'list', 'dataset', None),
# ('dataset', 'list', 'list', None),
# ('dataset', 'list', 'list:list', None),
# ('dataset', ['list', 'pair'], 'dataset', None),
# ('dataset', ['list', 'pair'], 'list', None),
# ('dataset', ['list', 'pair'], 'list:list', None),
("list", "dataset", "dataset", "list"),
("list", "dataset", "list", "list:list"),
("list", "dataset", "list:list", "list:list:list"),
("list", "dataset_multiple", "dataset", None),
("list", "dataset_multiple", "list", "list"),
("list", "dataset_multiple", "list:list", "list:list"),
("list", "list", "dataset", None),
("list", "list", "list", "list"),
("list", "list", "list:list", "list:list"),
("list", ["list", "pair"], "dataset", None),
("list", ["list", "pair"], "list", "list"),
("list", ["list", "pair"], "list:list", "list:list"),
("list:pair", "dataset", "dataset", "list:pair"),
("list:pair", "dataset", "list", "list:pair:list"),
("list:pair", "dataset", "list:list", "list:pair:list:list"),
# Pair into multiple="True" is not allowed
# ('list:pair', 'dataset_multiple', 'dataset', None),
# ('list:pair', 'dataset_multiple', 'list', None),
# ('list:pair', 'dataset_multiple', 'list:list', None),
# list:pair into list is not allowed
# ('list:pair', 'list', 'dataset', None),
# ('list:pair', 'list', 'list', None),
# ('list:pair', 'list', 'list:list', None),
("list:pair", ["list", "pair"], "dataset", "list"),
("list:pair", ["list", "pair"], "list", "list:list"),
("list:pair", ["list", "pair"], "list:list", "list:list:list"),
("list:list", "dataset", "dataset", "list:list"),
("list:list", "dataset", "list", "list:list:list"),
("list:list", "dataset", "list:list", "list:list:list:list"),
("list:list", "dataset_multiple", "dataset", "list"),
("list:list", "dataset_multiple", "list", "list:list"),
("list:list", "dataset_multiple", "list:list", "list:list:list"),
("list:list", "list", "dataset", "list"),
("list:list", "list", "list", "list:list"),
("list:list", "list", "list:list", "list:list:list"),
("list:list", ["list", "pair"], "dataset", "list"),
("list:list", ["list", "pair"], "list", "list:list"),
("list:list", ["list", "pair"], "list:list", "list:list:list"),
]
test_cases = []
for data_input, step_input_def, step_output_def, expected_collection_type in test_case_args:
steps: dict[int, dict[str, Any]] = {
0: _input_step(collection_type=data_input),
1: _output_step(step_input_def=step_input_def, step_output_def=step_output_def),
}
test_cases.append(
test_case(
data_input=data_input,
step_input_def=step_input_def,
step_output_def=step_output_def,
expected_collection_type=expected_collection_type,
steps=steps,
)
)
return test_cases
def _input_step(collection_type) -> dict[str, Any]:
output: dict[str, Any] = {"name": "output", "extensions": ["input_collection"]}
if collection_type != "dataset":
output["collection"] = True
output["collection_type"] = collection_type
step_type = "data_colletion_input" if collection_type == "dataset" else "data_input"
return {
"id": 0,
"type": step_type,
"inputs": [],
"outputs": [output],
"workflow_outputs": [],
"input_connections": {},
}
def _output_step(step_input_def, step_output_def) -> dict[str, Any]:
multiple = False
if step_input_def in ["dataset", "dataset_multiple"]:
input_type = "dataset"
collection_types = None
if step_input_def == "dataset_multiple":
multiple = True
else:
input_type = "dataset_collection"
collection_types = step_input_def if isinstance(step_input_def, list) else [step_input_def]
output: dict[str, Any] = {"name": "output", "extensions": ["data"]}
if step_output_def != "dataset":
output["collection"] = True
output["collection_type"] = step_output_def
input_connection_input: Any = [{"id": 0, "output_name": "output", "input_type": input_type}]
if step_input_def == "dataset":
# For whatever reason multiple = False inputs are not wrapped in a list.
input_connection_input = input_connection_input[0]
return {
"id": 1,
"type": "tool",
"inputs": [
{
"name": "input",
"multiple": multiple,
"input_type": input_type,
"collection_types": collection_types,
"extensions": ["data"],
}
],
"input_connections": {"input": input_connection_input},
"outputs": [output],
"workflow_outputs": [{"output_name": "output"}],
}
@pytest.mark.parametrize("test_case", _construct_steps_for_map_over())
def test_subworkflow_map_over_type(test_case):
trans = MockTrans()
new_steps = WorkflowContentsManager(app=trans.app, trs_proxy=trans.app.trs_proxy)._resolve_collection_type(
test_case.steps
)
assert (
new_steps[1]["outputs"][0].get("collection_type") == test_case.expected_collection_type
), "Expected collection_type '{}' for a '{}' input module, a '{}' input and a '{}' output, got collection_type '{}' instead".format(
test_case.expected_collection_type,
test_case.data_input,
test_case.step_input_def,
test_case.step_output_def,
new_steps[1]["outputs"][0].get("collection_type"),
)
def __populate_parameter_from_tool_form(multiple, default, parameter_type="integer"):
trans = MockTrans()
trans.workflow_building_mode = workflow_building_modes.ENABLED
module = modules.module_factory.from_dict(trans, {"type": "parameter_input"}, from_tool_form=True)
errors: dict[str, Any] = {}
incoming = {
"parameter_definition|parameter_type": parameter_type,
"parameter_definition|multiple": multiple,
"parameter_definition|optional|optional": "false",
"parameter_definition|optional|specify_default|specify_default": True,
"parameter_definition|optional|specify_default|default": default,
}
module.populate_state_from_tool_form(incoming, errors)
return module, errors
def __form_fields(inputs):
for field in inputs:
yield field
for case in field.get("cases", []):
yield from __form_fields(case["inputs"])
yield from __form_fields(field.get("inputs", []))
def __new_subworkflow_module(workflow=TEST_WORKFLOW_YAML):
trans = MockTrans()
mock_tool = __mock_tool(id="cat1", version="1.0")
trans.app.toolbox.tools["cat1"] = mock_tool
workflow = yaml_to_model(workflow)
stored_workflow = trans.save_workflow(workflow)
workflow_id = trans.app.security.encode_id(stored_workflow.id)
subworkflow_module = modules.module_factory.from_dict(trans, {"type": "subworkflow", "content_id": workflow_id})
return subworkflow_module
def __assert_has_runtime_input(module, label=None, collection_type=None):
test_step = getattr(module, "test_step", None)
if test_step is None:
test_step = mock.MagicMock()
inputs = module.get_runtime_inputs(test_step)
assert len(inputs) == 1
assert "input" in inputs
input_param = inputs["input"]
if label is not None:
assert input_param.get_label() == label, input_param.get_label()
if collection_type is not None:
assert input_param.collection_types == [collection_type]
return input_param
def __from_state(state):
trans = MockTrans()
module = modules.module_factory.from_dict(trans, state)
return module
def __from_step(**kwds):
if "trans" in kwds:
trans = kwds["trans"]
del kwds["trans"]
else:
trans = MockTrans()
step = __step(**kwds)
injector = modules.WorkflowModuleInjector(trans)
injector.inject(step, exact_tools=False)
injector.compute_runtime_state(step)
module = step.module
module.test_step = step
return module
def __numeric_tool_inputs(parameter_type, min, max, **tool_inputs):
return {
"parameter_type": parameter_type,
"optional": False,
"validators": [{"type": "in_range", "min": min, "max": max, "negate": False}],
**tool_inputs,
}
def __numeric_runtime_parameter(parameter_type, min, max):
trans = MockTrans()
trans.workflow_building_mode = workflow_building_modes.DISABLED
module = __from_step(
trans=trans,
type="parameter_input",
label="Count",
tool_inputs=__numeric_tool_inputs(parameter_type, min, max),
)
return module.get_runtime_inputs(module.test_step)["input"], trans
def __numeric_editor_module(parameter_type, min, max, **tool_inputs):
step = __step(
type="parameter_input",
label="Count",
tool_inputs=__numeric_tool_inputs(parameter_type, min, max, **tool_inputs),
)
return modules.module_factory.from_workflow_step(MockTrans(), step)
def __editor_default_parameter(module, parameter_type):
type_case = __case(module.get_inputs()["parameter_definition"], parameter_type)
specify_default = __case(type_case.inputs["optional"], "true").inputs["default"]
return __case(specify_default, "true").inputs["default"]
def __case(conditional, value):
return next(case for case in conditional.cases if case.value == value)
def __step(**kwds):
step = model.WorkflowStep()
for key, value in kwds.items():
setattr(step, key, value)
return step
def __mock_tool(
id="cat1",
version="1.0",
):
# For now ignoring inputs, params_from_strings, and
# check_and_update_param_values since only have unit tests for version
# handling - but need to write tests for all of this longer term.
tool = bunch.Bunch(
id=id,
version=version,
name=id,
inputs={},
outputs={
"out_file1": mock.Mock(
collection=None,
format="input",
format_source=None,
change_format=[],
filters=[],
label=None,
output_type="data",
spec=ToolOutput,
)
},
params_from_strings=mock.Mock(),
check_and_update_param_values=mock.Mock(),
to_json=_to_json,
)
return tool
def _to_json(*args, **kwargs):
return "{}"
# _mapped_inputs_from_collection_info: reduce a MatchingCollections to
# source-neutral per-input map-over descriptors. Workflow path is always
# linked=True.
def test_mapped_inputs_from_collection_info_none_or_empty():
assert modules._mapped_inputs_from_collection_info(None) == {}
assert modules._mapped_inputs_from_collection_info(bunch.Bunch(collections={})) == {}
def test_mapped_inputs_from_collection_info_hdca_no_subcollection():
collection_info = bunch.Bunch(collections={"a": bunch.Bunch(id=7)}, subcollection_types={})
mapped = modules._mapped_inputs_from_collection_info(collection_info)
assert set(mapped) == {"a"}
descriptor = mapped["a"]
assert descriptor.src == "hdca"
assert descriptor.id == 7
assert descriptor.map_over_type is None
assert descriptor.linked is True
def test_mapped_inputs_from_collection_info_subcollection_map_over_type():
collection_info = bunch.Bunch(
collections={"a": bunch.Bunch(id=7)},
subcollection_types={"a": bunch.Bunch(collection_type="paired")},
)
mapped = modules._mapped_inputs_from_collection_info(collection_info)
assert mapped["a"].map_over_type == "paired"
def test_mapped_inputs_from_collection_info_dce_src():
dce = mock.MagicMock(spec=model.DatasetCollectionElement)
dce.id = 9
collection_info = bunch.Bunch(collections={"a": dce}, subcollection_types={})
mapped = modules._mapped_inputs_from_collection_info(collection_info)
assert mapped["a"].src == "dce"
assert mapped["a"].id == 9
# _capture_workflow_tool_request_state outcome taxonomy: the function
# returns (template, combinations, tool_request). Skipped steps and
# unexpected capture-code defects return (None, None, None) — no
# ToolRequest minted. Real meta-model rejections after the converter has
# produced a structural payload mint a ToolRequest with
# request_state == "validation_failed". trans/step are unused here;
# collection_info=None -> no mapped inputs; resolve raises before any
# downstream is reached.
class _CaptureFakeTool:
id = "test_tool"
profile = "21.09"
parameters: list = []
def _capture(resolve):
# history=None means the mint path is unreachable in these unit cases;
# every exercised branch returns before touching it.
return modules._capture_workflow_tool_request_state(None, _CaptureFakeTool(), None, None, None, resolve, [])
def test_capture_skipped_conditional_step_returns_none():
"""A falsy `when` raises SkipWorkflowStepEvaluation: nothing to capture."""
def resolve(iteration_elements):
raise modules.SkipWorkflowStepEvaluation
assert _capture(resolve) == (None, None, None)
def test_capture_converter_guard_returns_none_quietly():
"""Converter raised before request_internal was built: no ToolRequest, quiet."""
def resolve(iteration_elements):
raise modules.RequestInternalToWorkflowStateError("cross-product")
with mock.patch.object(modules, "log") as log:
assert _capture(resolve) == (None, None, None)
log.debug.assert_called_once()
log.warning.assert_not_called()
def test_capture_invalid_state_returns_none_quietly():
"""Meta-model rejection at resolve-time: no payload to record, quiet."""
def resolve(iteration_elements):
raise modules.exceptions.RequestParameterInvalidException("bad state")
with mock.patch.object(modules, "log") as log:
assert _capture(resolve) == (None, None, None)
log.debug.assert_called_once()
log.warning.assert_not_called()
def test_capture_unexpected_error_returns_none_loudly():
"""Capture-code defect: drop the partial payload, surface at warning."""
def resolve(iteration_elements):
raise RuntimeError("capture bug")
with mock.patch.object(modules, "log") as log:
assert _capture(resolve) == (None, None, None)
log.warning.assert_called_once()
class _PersistableFakeTool(_CaptureFakeTool):
version = "1.0"
dynamic_tool = None
tool_source = bunch.Bunch(to_string=lambda: "<tool/>")
def test_capture_persists_request_for_implicit_false_branch():
"""An omitted false XML branch must not prevent mapped workflow request capture."""
tool = _PersistableFakeTool()
tool.tool_source = parameter_tool_source("gx_conditional_boolean_implicit")
tool.parameters = input_models_for_tool_source(tool.tool_source).parameters
trans = MockTrans()
history = model.History(name="target")
trans.sa_session.add(history)
collection_info = bunch.Bunch(collections={"input": bunch.Bunch(id=7)}, subcollection_types={})
job_state = {"input": {"src": "hda", "id": 13}, "conditional_parameter": {"test_parameter": False}}
def resolve(iteration_elements):
return (
bunch.Bunch(
inputs={
"input": None,
"conditional_parameter": {"test_parameter": False, "__current_case__": 1},
}
),
None,
None,
)
template, combinations, request = modules._capture_workflow_tool_request_state(
trans, tool, None, collection_info, history, resolve, [job_state]
)
assert request is not None
assert request.id is not None
assert request.request_state == "validated"
assert request.request == {
"input": {"__class__": "Batch", "values": [{"src": "hdca", "id": 7}], "linked": True},
"conditional_parameter": {"test_parameter": False},
}
assert template is not None
assert template.input_state == request.request
assert combinations is not None
assert [combination.input_state for combination in combinations] == [job_state]
def test_capture_propagates_failure_flushing_unrelated_pending_objects():
trans = MockTrans()
history = model.History(name="target")
existing = model.History(name="existing")
trans.sa_session.add_all([history, existing])
trans.sa_session.commit()
existing_id = existing.id
trans.sa_session.expunge(existing)
trans.sa_session.add(model.History(id=existing_id, name="duplicate"))
def resolve(iteration_elements):
return bunch.Bunch(inputs={}), None, None
with pytest.raises(IntegrityError):
modules._capture_workflow_tool_request_state(trans, _PersistableFakeTool(), None, None, history, resolve, [])
def test_to_decoded_json_lowers_connected_value_in_repeat():
"""ConnectedValue inside a repeat lowers to its JSON marker.
Regression: an unresolved connection inside a `repeat` survived to the
tool_request flush as a raw ``ConnectedValue()`` and broke JSON encode
with a TypeError. ``to_decoded_json`` recurses and lowers it to a marker.
"""
payload = {
"datasets": [
{"input": {"src": "hda", "id": 1}},
{"input": ConnectedValue()},
],
}
result = to_decoded_json(payload)
assert result["datasets"][0]["input"] == {"src": "hda", "id": 1}
assert result["datasets"][1]["input"] == {"__class__": "ConnectedValue"}
json.dumps(result)
def test_to_decoded_json_lowers_bare_runtime_value():
"""Bare RuntimeValue tokens lower to their JSON marker form too."""
result = to_decoded_json({"foo": RuntimeValue()})
assert result == {"foo": {"__class__": "RuntimeValue"}}
json.dumps(result)