Files
galaxy/test/integration/test_scripts_pgcleanup.py

590 lines
24 KiB
Python

import datetime
import os
import string
import tempfile
from pathlib import Path
from typing import Any
import pytest
from sqlalchemy import select
from galaxy import model
from galaxy.util import now
from galaxy_test.base.populators import (
LibraryPopulator,
skip_without_tool,
)
from galaxy_test.driver import integration_util
from .test_job_cache_metadata_files import RunsCachedBamJob
from .test_scripts import BaseScriptsIntegrationTestCase
DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE = string.Template("""
type: distributed
backends:
- type: disk
id: default
weight: 1
name: Default Store
files_dir: "${temp_directory}/files_default"
extra_dirs:
- type: temp
path: "${temp_directory}/tmp_default"
- type: job_work
path: "${temp_directory}/job_working_directory_default"
""")
USER_OBJECT_STORE_CATALOG = """
- id: general_disk
name: General Disk
description: General Disk Bound to You
configuration:
type: disk
files_dir: '/data/general/{{ user.username }}'
"""
SCRIPT = "cleanup_datasets/pgcleanup.py"
class TestScriptsPgCleanupIntegration(RunsCachedBamJob, BaseScriptsIntegrationTestCase):
def setUp(self):
super().setUp()
self.library_populator = LibraryPopulator(self.galaxy_interactor)
def _db_get(self, model_class, encoded_id: str) -> Any:
"""Fetch a model instance reflecting state committed by the pgcleanup subprocess.
The session is rolled back first so the identity map is not used as a cache of state
that predates the subprocess' writes.
"""
sa_session = self._app.model.session
sa_session.rollback()
return sa_session.get(model_class, self._decode_id(encoded_id))
def _library_dataset_state(self, library_dataset: dict[str, Any]) -> dict[str, bool]:
"""Return the cleanup flags of a library dataset, its current version, and its dataset."""
ld = self._db_get(model.LibraryDataset, library_dataset["id"])
ldda = self._db_get(model.LibraryDatasetDatasetAssociation, library_dataset["ldda_id"])
dataset = self._app.model.session.get(model.Dataset, ldda.dataset_id)
assert dataset is not None
return {
"library_dataset_deleted": bool(ld.deleted),
"library_dataset_purged": bool(ld.purged),
"version_deleted": bool(ldda.deleted),
"dataset_deleted": bool(dataset.deleted),
"dataset_purged": bool(dataset.purged),
}
def _pgcleanup_action_log(self, extra_args: list[str], action: str) -> str:
"""Run pgcleanup and return the named action's log; empty if it logged no rows."""
with tempfile.TemporaryDirectory() as log_dir:
self._pgcleanup_check_output(["--log-dir", log_dir] + extra_args)
log_path = os.path.join(log_dir, f"{action}.log")
if not os.path.exists(log_path):
return ""
with open(log_path) as f:
return f.read()
def test_help(self):
self._skip_unless_postgres()
self._scripts_check_argparse_help(SCRIPT)
def test_purge_deleted_histories(self):
self._skip_unless_postgres()
history_id = self.dataset_populator.new_history()
delete_response = self.dataset_populator._delete(f"histories/{history_id}")
assert delete_response.status_code == 200
assert delete_response.json()["purged"] is False
self._pgcleanup_check_output(["--older-than", "0", "--sequence", "purge_deleted_histories"])
history_response = self.dataset_populator._get(f"histories/{history_id}")
assert history_response.status_code == 200
assert history_response.json()["purged"] is True, history_response.json()
def test_purge_old_hdas(self):
self._skip_unless_postgres()
history_id = self.dataset_populator.new_history()
hda = self.dataset_populator.new_dataset(history_id, wait=True)
assert not self.is_purged(history_id, hda)
# filtering on a date too old - shouldn't purge the dataset
self._pgcleanup_check_output(
[
"--older-than",
"1",
"--sequence",
"purge_old_hdas",
]
)
assert not self.is_purged(history_id, hda)
# filtering on invalid object store - shouldn't purge the dataset
self._pgcleanup_check_output(
[
"--older-than",
"0",
"--object-store-id",
"myfakeobjectstore",
"--sequence",
"purge_old_hdas",
]
)
assert not self.is_purged(history_id, hda)
self._pgcleanup_check_output(["--older-than", "0", "--sequence", "purge_old_hdas"])
assert self.is_purged(history_id, hda)
@skip_without_tool("test_data_source")
def test_purge_errored_hdas(self):
self._skip_unless_postgres()
history_id = self.dataset_populator.new_history()
error_dataset = self.dataset_populator.new_error_dataset(history_id)
assert not self.is_purged(history_id, error_dataset)
# dataset not old enough, shouldn't be purged
self._pgcleanup_check_output(
[
"--older-than",
"1",
"--sequence",
"purge_error_hdas",
]
)
assert not self.is_purged(history_id, error_dataset)
# dataset not in target object store, shouldn't be purged
self._pgcleanup_check_output(
[
"--older-than",
"0",
"--object-store-id",
"myfakeobjectstore",
"--sequence",
"purge_error_hdas",
]
)
assert not self.is_purged(history_id, error_dataset)
# okay though, this should purge the dataset
self._pgcleanup_check_output(
[
"--older-than",
"0",
"--sequence",
"purge_error_hdas",
]
)
assert self.is_purged(history_id, error_dataset)
def test_purge_datasets(self):
self._skip_unless_postgres()
history_id = self.dataset_populator.new_history()
hda = self.dataset_populator.new_dataset(history_id, wait=True)
self.dataset_populator.delete_dataset(history_id, hda["id"])
assert not self.is_purged(history_id, hda)
self._pgcleanup_check_output(
[
"--older-than",
"1",
"--sequence",
"purge_datasets",
]
)
assert not self.is_purged(history_id, hda)
self._pgcleanup_check_output(
[
"--older-than",
"0",
"--object-store-id",
"myfakeobjectstore",
"--sequence",
"purge_datasets",
]
)
assert not self.is_purged(history_id, hda)
self._pgcleanup_check_output(
[
"--older-than",
"0",
"--sequence",
"purge_datasets",
]
)
self._pgcleanup_check_output(
[
"--older-than",
"0",
"--sequence",
"purge_datasets",
]
)
# why is this not purged?
# test or functionality seem broken but better to run through it and ensure
# it isn't breaking anything and everything is syntactically correct than not
# assert self.is_purged(history_id, hda)
def test_delete_datasets(self):
# this walks through the code to ensure no SQL or Python errors but
# I think we would need to talk to the model layer from the test directly
# to actually produce datasets of the target type for purging and to verify
# they were purged (certainly a possibility)
self._skip_unless_postgres()
history_id = self.dataset_populator.new_history()
hda = self.dataset_populator.new_dataset(history_id, wait=True)
assert not self.is_purged(history_id, hda)
self._pgcleanup_check_output(
[
"--older-than",
"0",
"--sequence",
"delete_datasets",
]
)
self._pgcleanup_check_output(
[
"--older-than",
"0",
"--object-store-id",
"myfakeobjectstore",
"--sequence",
"delete_datasets",
]
)
assert not self.is_purged(history_id, hda)
def test_delete_library_datasets(self):
self._skip_unless_postgres()
_, library_dataset = self.library_populator.new_library_dataset_in_private_library("pgcleanup_library_dataset")
# a library dataset that was never deleted is not touched
self._pgcleanup_check_output(["--older-than", "0", "delete_library_datasets"])
assert self._library_dataset_state(library_dataset)["version_deleted"] is False
# deleting a library dataset through the API only flags the library dataset, its versions
# (and therefore the dataset they reference) are left alone - which is why datasets that
# only live in libraries are never deleted or purged by any other cleanup action
self.library_populator.delete_library_dataset(library_dataset["id"])
assert self._library_dataset_state(library_dataset) == {
"library_dataset_deleted": True,
"library_dataset_purged": False,
"version_deleted": False,
"dataset_deleted": False,
"dataset_purged": False,
}
# the library dataset is not old enough to be acted on
self._pgcleanup_check_output(["--older-than", "1", "delete_library_datasets"])
assert self._library_dataset_state(library_dataset)["version_deleted"] is False
# with no retention window the version is marked deleted and the library dataset purged
self._pgcleanup_check_output(["--older-than", "0", "delete_library_datasets"])
assert self._library_dataset_state(library_dataset) == {
"library_dataset_deleted": True,
"library_dataset_purged": True,
"version_deleted": True,
"dataset_deleted": False,
"dataset_purged": False,
}
sa_session = self._app.model.session
ld_events = sa_session.scalars(
select(model.CleanupEventLibraryDatasetAssociation.cleanup_event_id).filter_by(
library_dataset_id=self._decode_id(library_dataset["id"])
)
).all()
ldda_events = sa_session.scalars(
select(model.CleanupEventLibraryDatasetDatasetAssociationAssociation.cleanup_event_id).filter_by(
ldda_id=self._decode_id(library_dataset["ldda_id"])
)
).all()
assert len(ld_events) == 1
assert ldda_events == ld_events
# chaining the pre-existing actions now reaches the dataset itself
self._pgcleanup_check_output(
["--older-than", "0", "delete_library_datasets", "delete_datasets", "purge_datasets"]
)
assert self._library_dataset_state(library_dataset) == {
"library_dataset_deleted": True,
"library_dataset_purged": True,
"version_deleted": True,
"dataset_deleted": True,
"dataset_purged": True,
}
def test_delete_library_datasets_chains_with_retention_window(self):
self._skip_unless_postgres()
_, library_dataset = self.library_populator.new_library_dataset_in_private_library(
"pgcleanup_library_dataset_retention"
)
self.library_populator.delete_library_dataset(library_dataset["id"])
two_days_ago = now() - datetime.timedelta(days=2)
ld = self._db_get(model.LibraryDataset, library_dataset["id"])
ldda = self._db_get(model.LibraryDatasetDatasetAssociation, library_dataset["ldda_id"])
ld.update_time = two_days_ago
ldda.update_time = two_days_ago
self._app.model.session.commit()
self._pgcleanup_check_output(
["--older-than", "1", "delete_library_datasets", "delete_datasets", "purge_datasets"]
)
# purge_datasets waits a retention window on the update_time delete_datasets just stamped
assert self._library_dataset_state(library_dataset) == {
"library_dataset_deleted": True,
"library_dataset_purged": True,
"version_deleted": True,
"dataset_deleted": True,
"dataset_purged": False,
}
ldda = self._db_get(model.LibraryDatasetDatasetAssociation, library_dataset["ldda_id"])
assert ldda.update_time == two_days_ago
def test_delete_library_datasets_keeps_dataset_shared_with_history(self):
self._skip_unless_postgres()
_, library_dataset = self.library_populator.new_library_dataset_in_private_library(
"pgcleanup_library_dataset_shared"
)
history_id = self.dataset_populator.new_history()
import_response = self._post(
f"histories/{history_id}/contents",
data={"source": "library", "content": library_dataset["id"], "type": "dataset"},
json=True,
)
self._assert_status_code_is_ok(import_response)
hda = import_response.json()
hda_model = self._db_get(model.HistoryDatasetAssociation, hda["id"])
ldda = self._db_get(model.LibraryDatasetDatasetAssociation, library_dataset["ldda_id"])
assert hda_model.dataset_id == ldda.dataset_id
self.library_populator.delete_library_dataset(library_dataset["id"])
chain = ["--older-than", "0", "delete_library_datasets", "delete_datasets", "purge_datasets"]
self._pgcleanup_check_output(chain)
assert self._library_dataset_state(library_dataset) == {
"library_dataset_deleted": True,
"library_dataset_purged": True,
"version_deleted": True,
"dataset_deleted": False,
"dataset_purged": False,
}
assert not self.is_purged(history_id, hda)
# once the history copy is purged too, the dataset goes
self.dataset_populator.delete_dataset(history_id, hda["id"], purge=True, wait_for_purge=True)
assert self._library_dataset_state(library_dataset)["dataset_deleted"] is False
self._pgcleanup_check_output(chain)
state = self._library_dataset_state(library_dataset)
assert state["dataset_deleted"] is True
assert state["dataset_purged"] is True
def test_delete_library_datasets_force_retry(self):
self._skip_unless_postgres()
_, library_dataset = self.library_populator.new_library_dataset_in_private_library(
"pgcleanup_library_dataset_force_retry"
)
self.library_populator.delete_library_dataset(library_dataset["id"])
self._pgcleanup_check_output(
["--older-than", "0", "delete_library_datasets", "delete_datasets", "purge_datasets"]
)
assert self._library_dataset_state(library_dataset)["dataset_purged"] is True
# simulate a version left behind by the legacy scripts: the dataset is already purged but the
# association itself was never marked deleted
ldda = self._db_get(model.LibraryDatasetDatasetAssociation, library_dataset["ldda_id"])
ldda.deleted = False
self._app.model.session.commit()
assert self._library_dataset_state(library_dataset)["version_deleted"] is False
# purged library datasets are skipped by default...
self._pgcleanup_check_output(["--older-than", "0", "delete_library_datasets"])
assert self._library_dataset_state(library_dataset)["version_deleted"] is False
# ...but --force-retry revisits them to reconcile the missing flags
logged = f"library_dataset_id: {self._decode_id(library_dataset['id'])}\n"
force_retry = ["--older-than", "0", "--force-retry", "delete_library_datasets"]
assert logged in self._pgcleanup_action_log(force_retry, "delete_library_datasets")
assert self._library_dataset_state(library_dataset)["version_deleted"] is True
# a reconciled library dataset is not reported again
assert logged not in self._pgcleanup_action_log(force_retry, "delete_library_datasets")
def test_delete_library_datasets_force_retry_null_purged(self):
self._skip_unless_postgres()
_, library_dataset = self.library_populator.new_library_dataset_in_private_library(
"pgcleanup_library_dataset_null_purged"
)
self.library_populator.delete_library_dataset(library_dataset["id"])
# legacy rows may carry a NULL purged flag
ld = self._db_get(model.LibraryDataset, library_dataset["id"])
ld.purged = None
self._app.model.session.commit()
self._pgcleanup_check_output(["--older-than", "0", "--force-retry", "delete_library_datasets"])
state = self._library_dataset_state(library_dataset)
assert state["version_deleted"] is True
assert state["library_dataset_purged"] is True
def test_purge_historyless_hdas(self):
# same as above - this is just a negative test for things being broken
# we could access the model layer to write a test to verify the positive
# behavior actually occurs
self._skip_unless_postgres()
history_id = self.dataset_populator.new_history()
hda = self.dataset_populator.new_dataset(history_id, wait=True)
assert not self.is_purged(history_id, hda)
self._pgcleanup_check_output(
[
"--older-than",
"0",
"--sequence",
"purge_historyless_hdas",
]
)
self._pgcleanup_check_output(
[
"--older-than",
"0",
"--object-store-id",
"myfakeobjectstore",
"--sequence",
"purge_historyless_hdas",
]
)
assert not self.is_purged(history_id, hda)
@pytest.mark.parametrize("rename_input", [False, True], ids=["hit_at_submission", "hit_at_job_preparation"])
def test_purge_deleted_hdas_with_cached_copy(self, rename_input: bool):
self._skip_unless_postgres()
history_id = self.dataset_populator.new_history()
source_id, cached_id = self._run_cached_bam_job(history_id, rename_input=rename_input)
cached = self._hda(cached_id)
cached_index_path = cached.metadata.bam_index.get_file_name()
cached_index = Path(cached_index_path).read_bytes()
assert cached_index
self.dataset_populator.delete_dataset(history_id, source_id)
self._pgcleanup_check_output(["--older-than", "0", "--sequence", "purge_deleted_hdas"])
assert self.is_purged(history_id, source_id)
assert not self.is_purged(history_id, cached_id)
assert Path(cached_index_path).read_bytes() == cached_index
class TestPgCleanupUserObjectStoreIntegration(
BaseScriptsIntegrationTestCase,
integration_util.ConfiguresObjectStores,
):
"""Test that pgcleanup handles datasets in user-defined object stores gracefully.
When a dataset is stored in a user-defined object store (object_store_id starting
with "user_objects://"), the pgcleanup script cannot resolve the object store backend
since it doesn't have access to the UserObjectStoreResolver. The script should skip
physical file removal for such objects while still updating the database records.
"""
framework_tool_and_types = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
super().handle_galaxy_config_kwds(config)
cls._configure_object_store(DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE, config, format="yml")
config["object_store_store_by"] = "uuid"
cls._configure_object_store_template_catalog(USER_OBJECT_STORE_CATALOG, config)
def _create_user_object_store(self) -> str:
"""Create a user-defined object store via the API and return its object_store_id."""
body = {
"name": "My Test Disk",
"template_id": "general_disk",
"template_version": 0,
"secrets": {},
"variables": {},
}
object_store_json = self.dataset_populator.create_object_store(body)
object_store_id = object_store_json["object_store_id"]
assert object_store_id.startswith("user_objects://")
return object_store_id
def test_purge_deleted_histories_with_user_object_store_dataset(self):
"""Test that purging histories containing datasets in user object stores doesn't crash.
This verifies the fix for the KeyError that occurred when pgcleanup tried to call
get_store_by() on objects stored in user-defined object stores, which cannot be
resolved without a UserObjectStoreResolver (not available in the cleanup script).
"""
self._skip_unless_postgres()
# Create a user-defined object store
object_store_id = self._create_user_object_store()
# Set the user's preferred object store to the user-defined one
self.dataset_populator.set_user_preferred_object_store_id(object_store_id)
# Create a history and upload a dataset (it should go to the user object store)
history_id = self.dataset_populator.new_history()
hda = self.dataset_populator.new_dataset(history_id, content="test data for user object store", wait=True)
# Verify the dataset is in the user object store
storage_info = self.dataset_populator.dataset_storage_info(hda["id"])
assert (
storage_info["object_store_id"] == object_store_id
), f"Expected dataset in user object store {object_store_id}, but got {storage_info['object_store_id']}"
# Reset user preference so subsequent datasets don't use it
self.dataset_populator.set_user_preferred_object_store_id(None)
# Delete the history
delete_response = self.dataset_populator._delete(f"histories/{history_id}")
assert delete_response.status_code == 200
assert delete_response.json()["purged"] is False
# Run pgcleanup - this should NOT crash with a KeyError
# Before the fix, this would raise:
# KeyError: 'user_objects://...' in _resolve_backend
self._pgcleanup_check_output(["--older-than", "0", "--sequence", "purge_deleted_histories"])
# The history should be purged in the database even though the physical
# file in the user object store could not be removed
history_response = self.dataset_populator._get(f"histories/{history_id}")
assert history_response.status_code == 200
assert history_response.json()["purged"] is True, history_response.json()
def test_purge_old_hdas_with_user_object_store_dataset(self):
"""Test that purging HDAs in user object stores doesn't crash pgcleanup."""
self._skip_unless_postgres()
object_store_id = self._create_user_object_store()
self.dataset_populator.set_user_preferred_object_store_id(object_store_id)
history_id = self.dataset_populator.new_history()
hda = self.dataset_populator.new_dataset(history_id, content="test data", wait=True)
# Verify the dataset is in the user object store
storage_info = self.dataset_populator.dataset_storage_info(hda["id"])
assert storage_info["object_store_id"] == object_store_id
self.dataset_populator.set_user_preferred_object_store_id(None)
# Run pgcleanup - should not crash
self._pgcleanup_check_output(["--older-than", "0", "--sequence", "purge_old_hdas"])
# The HDA should be purged in the database
assert self.is_purged(history_id, hda)