diff --git a/.github/workflows/python_unit_tests.yaml b/.github/workflows/python_unit_tests.yaml new file mode 100644 index 00000000..7d1465b5 --- /dev/null +++ b/.github/workflows/python_unit_tests.yaml @@ -0,0 +1,35 @@ +name: Python Unit Tests + +on: + pull_request: + workflow_dispatch: + +permissions: read-all + +jobs: + python_unit_tests: + name: python_unit_tests + runs-on: ubuntu-24.04 + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v5 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.14' + - name: Install dependencies + run: | + # We only have 1 external dependency other than pytest for now, so + # list them here + # If this changes, we may want to switch to a dependencies file of + # some format + python -m pip install --upgrade pip + pip install pytest + pip install networkx + pip install PyYAML + - name: Test with pytest + run: | + git config --global user.name 'Testing' + git config --global user.email 'Testing' + pytest -vv diff --git a/README.md b/README.md index 8ffcdb34..ed3cb1fc 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![Checks](https://github.com/MetOffice/SimSys_Scripts/actions/workflows/lint.yml/badge.svg)](https://github.com/MetOffice/SimSys_Scripts/actions/workflows/lint.yml) [![CodeQL](https://github.com/MetOffice/SimSys_Scripts/actions/workflows/github-code-scanning/codeql/badge.svg)](https://github.com/MetOffice/SimSys_Scripts/actions/workflows/github-code-scanning/codeql) +[![Python Unit Tests](https://github.com/MetOffice/SimSys_Scripts/actions/workflows/python_unit_tests.yaml/badge.svg)](https://github.com/MetOffice/SimSys_Scripts/actions/workflows/python_unit_tests.yaml) This repository contains support scripts that are common across the many simulation and modelling codes owned by the Met Office. Particularly those diff --git a/github_scripts/get_git_sources.py b/github_scripts/get_git_sources.py index 08efaa0e..6b4d26ca 100644 --- a/github_scripts/get_git_sources.py +++ b/github_scripts/get_git_sources.py @@ -60,24 +60,27 @@ def validate_dependencies(dependencies: dict) -> None: Each dictionary value should be a list of dictionaries (or a single dictionary) Those dictionaries should have a "source" and a "ref" key """ + if not isinstance(dependencies, dict): + raise TypeError( + "The dependencies object should be a dict with keys as source repositories" + ) for item, values in dependencies.items(): - failed = False + err_message = ( + f"The dependency {item} does not contain a list of dictionaries (or a " + "single dictionary) with keys of 'source' and 'ref'.\nPlease edit your " + "dependencies.yaml file to satisfy this." + ) + if isinstance(values, dict): values = [values] if not isinstance(values, list): - failed = True - else: - for entry in values: - if not isinstance(entry, dict) or ( - "source" not in entry or "ref" not in entry - ): - failed = True - if failed: - raise ValueError( - f"The dependency {item} does not contain a list of dictionaries (or a " - "single dictionary) with keys of 'source' and 'ref'.\nPlease edit your " - "dependencies.yaml file to satisfy this." - ) + raise TypeError(err_message) + + for entry in values: + if not isinstance(entry, dict): + raise TypeError(err_message) + if "source" not in entry or "ref" not in entry: + raise ValueError(err_message) def datetime_str() -> str: @@ -88,7 +91,11 @@ def datetime_str() -> str: def clone_and_merge( - dependency: str, opts: Union[list, dict], loc: Path, use_mirrors: bool, mirror_loc: Path + dependency: str, + opts: Union[list, dict], + loc: Path, + use_mirrors: bool, + mirror_loc: Path, ) -> None: """ Wrapper script for calling get_source and merge_source for a single dependency @@ -160,7 +167,7 @@ def get_source( def merge_source( - source: str, + source: Union[Path, str], ref: str, dest: Path, repo: str, @@ -214,6 +221,9 @@ def handle_merge_conflicts(source: str, ref: str, loc: Path, dependency: str) -> # For suites, merge conflicts in these files/directories are unimportant so accept # the current changes for filepath in ("dependencies.yaml", "rose-stem"): + full_path = loc / filepath + if not full_path.exists(): + continue logger.warning(f"Ignoring merge conflicts in {filepath}") run_command(f"git -C {loc} checkout --ours -- {filepath}") run_command(f"git -C {loc} add {filepath}") @@ -239,6 +249,20 @@ def get_unmerged(loc: Path) -> list[str]: return files.stdout.split() +def check_existing(loc: Path) -> None: + """ + If the repository exists and isn't a git repo, exit now as we don't want to + overwrite it + """ + + if loc.exists(): + if not Path(loc / ".git").exists(): + raise FileExistsError( + f"The destination, '{loc}', already exists but isn't a git directory. " + "Exiting so as to not overwrite it." + ) + + def clone_repo_mirror( repo_source: str, repo_ref: str, @@ -254,15 +278,8 @@ def clone_repo_mirror( - loc: path to clone the repository to """ - # If the repository exists and isn't a git repo, exit now as we don't want to - # overwrite it if loc.exists(): - if not Path(loc / ".git").exists(): - raise RuntimeError( - f"The destination for the clone of {repo_source} already exists but " - "isn't a git directory. Exiting so as to not overwrite it." - ) - + check_existing(loc) # Clone if the repo doesn't exist else: command = f"git clone {mirror_loc} {loc}" @@ -288,6 +305,7 @@ def determine_mirror_fetch(repo_source: str, repo_ref: str) -> str: """ repo_source = repo_source.removeprefix("git@github.com:") + repo_source = repo_source.removeprefix("https://github.com/") user = repo_source.split("/")[0] # Check that the user is different to the Upstream User if "MetOffice" in user: @@ -328,6 +346,7 @@ def clone_repo(repo_source: str, repo_ref: str, loc: Path) -> None: for command in commands: run_command(command) else: + check_existing(loc) commands = ( f"git -C {loc} fetch origin {repo_ref}", f"git -C {loc} checkout FETCH_HEAD", @@ -336,11 +355,13 @@ def clone_repo(repo_source: str, repo_ref: str, loc: Path) -> None: run_command(command) -def sync_repo(repo_source: str, repo_ref: str, loc: Path) -> None: +def sync_repo(repo_source: Union[str, Path], repo_ref: str, loc: Path) -> None: """ Rsync a local git clone and checkout the provided ref """ + repo_source = str(repo_source) + # Remove if this clone already exists if loc.exists(): rmtree(loc) diff --git a/github_scripts/tests/test_get_git_sources.py b/github_scripts/tests/test_get_git_sources.py new file mode 100644 index 00000000..ba0e1787 --- /dev/null +++ b/github_scripts/tests/test_get_git_sources.py @@ -0,0 +1,259 @@ +# ----------------------------------------------------------------------------- +# (C) Crown copyright Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +# ----------------------------------------------------------------------------- +""" +Unit tests for get_git_sources +""" + +import os +import subprocess +from shlex import split +from pathlib import Path +import pytest + +from ..get_git_sources import ( + validate_dependencies, + determine_mirror_fetch, + set_https, + clone_repo, + clone_repo_mirror, + sync_repo, + check_existing, + merge_source, +) + + +@pytest.fixture(scope="session") +def setup_sources(tmpdir_factory): + """ + Setup a tempdir for cloning into, a mirror of SimSys_Scripts and a local clone of + SimSys_Scripts + Use SimSys_Scripts as a public repo + """ + + location = tmpdir_factory.mktemp("data") + os.chdir(location) + + # Setup local mirror + subprocess.run( + split("git clone --mirror https://github.com/MetOffice/SimSys_Scripts.git"), + check=True, + ) + + # Create local clone + subprocess.run( + split("git clone https://github.com/MetOffice/SimSys_Scripts.git"), check=True + ) + subprocess.run(split("git -C SimSys_Scripts checkout 2025.12.1")) + + # Create a non-git repo to test check_existing + existing = Path(location) / "empty_dir" + existing.mkdir() + + # Create 2 clones with conflicting commits + for i in range(2): + subprocess.run(split(f"cp -r SimSys_Scripts merge{i}"), check=True) + subprocess.run(split(f"git -C merge{i} switch -c merge{i}"), check=True) + with open(f"merge{i}/merge.txt", "w") as f: + f.write(f"merge{i}") + subprocess.run(split(f"git -C merge{i} add merge.txt"), check=True) + subprocess.run( + split(f"git -C merge{i} commit -a -m 'merge conflict'"), + check=True, + ) + + return Path(location) + + +def test_clone_repo(setup_sources): + """ + Test cloning from a github source + """ + + output_loc = setup_sources / "github_clone" + assert ( + clone_repo( + "https://github.com/MetOffice/SimSys_Scripts.git", "2025.12.1", output_loc + ) + is None + ) + assert Path(output_loc / ".git").is_dir() is True + + +def test_clone_repo_mirror(setup_sources): + """ + Test rsyncing a local clone + """ + + output_loc = setup_sources / "mirror_clone" + mirror_loc = setup_sources / "SimSys_Scripts.git" + assert ( + clone_repo_mirror( + "https://github.com/MetOffice/SimSys_Scripts.git", + "2025.12.1", + mirror_loc, + output_loc, + ) + is None + ) + assert Path(output_loc / ".git").is_dir() is True + + +def test_sync_repo(setup_sources): + """ + Test cloning from a github source + """ + + # Cant test syncing with a hostname in github actions + source_loc = setup_sources / "SimSys_Scripts" + output_loc = setup_sources / "sync_clone" + assert sync_repo(source_loc, "2025.12.1", output_loc) is None + assert Path(output_loc / ".git").is_dir() is True + + +def test_merge_sources(setup_sources): + """ + Test merge_source + """ + + target_clone = setup_sources / "SimSys_Scripts" + + assert ( + merge_source( + "https://github.com/MetOffice/SimSys_Scripts.git", + "main", + target_clone, + "SimSys_Scripts", + ) + is None + ) + assert ( + merge_source(setup_sources / "merge0", "merge0", target_clone, "SimSys_Scripts") + is None + ) + with pytest.raises(RuntimeError): + merge_source(setup_sources / "merge1", "merge1", target_clone, "SimSys_Scripts") + + +def test_check_exists(setup_sources): + """ + Test check_existing + """ + + assert check_existing(setup_sources / "SimSys_Scripts") is None + + with pytest.raises(FileExistsError): + check_existing(setup_sources / "empty_dir") + + +def test_validate_dependencies(): + valid = { + "repo1": {"source": "abc", "ref": "123"}, + "repo2": [{"source": "abc", "ref": "123"}, {"source": "abc", "ref": "123"}], + } + assert validate_dependencies(valid) is None + + invalid_dependencies = set() + invalid_dependencies.add(1) + with pytest.raises(TypeError): + validate_dependencies(invalid_dependencies) + + invalid_repo_type = {"repo1": invalid_dependencies} + with pytest.raises(TypeError): + validate_dependencies(invalid_repo_type) + + invalid_list = {"repo1": [invalid_dependencies]} + with pytest.raises(TypeError): + validate_dependencies(invalid_list) + + missing_source = {"repo1": {"ref": "123"}} + with pytest.raises(ValueError): + validate_dependencies(missing_source) + + missing_ref = {"repo1": {"source": "abc"}} + with pytest.raises(ValueError): + validate_dependencies(missing_ref) + + +def test_determine_mirror_fetch(): + """ + Test determine_mirror_fetch + """ + + # Test MetOffice User + assert ( + determine_mirror_fetch("git@github.com:MetOffice/SimSys_Scripts.git", "ref") + == "ref" + ) + assert ( + determine_mirror_fetch("https://github.com/MetOffice/SimSys_Scripts.git", "ref") + == "ref" + ) + + # Test using hash + commit_hash = "ba965768395de47de064a60ee769471e3868e02d" + assert ( + determine_mirror_fetch( + "git@github.com:user_name/SimSys_Scripts.git", commit_hash + ) + == commit_hash + ) + assert ( + determine_mirror_fetch( + "https://github.com/user_name/SimSys_Scripts.git", commit_hash + ) + == commit_hash + ) + + # Test using user and branch + user_name = "user_name" + branch = "branch" + assert ( + determine_mirror_fetch(f"git@github.com:{user_name}/SimSys_Scripts.git", branch) + == f"{user_name}/{branch}" + ) + assert ( + determine_mirror_fetch( + f"https://github.com/{user_name}/SimSys_Scripts.git", branch + ) + == f"{user_name}/{branch}" + ) + + +def test_set_https(): + """ + Test set_https + """ + + input_dict = { + "repo1": { + "source": "git@github.com:MetOffice/SimSys_Scripts.git", + "ref": "123", + }, + "repo2": [ + {"source": "git@github.com:MetOffice/lfric_apps.git", "ref": "123"}, + {"source": "git@github.com:MetOffice/lfric_apps.git", "ref": "456"}, + ], + "repo3": { + "source": "https://github.com/MetOffice/lfric_core.git", + "ref": "123", + }, + "repo4": {"source": "hostname:/path/to/repository", "ref": "123"}, + } + output_dict = { + "repo1": [ + {"source": "https://github.com/MetOffice/SimSys_Scripts.git", "ref": "123"} + ], + "repo2": [ + {"source": "https://github.com/MetOffice/lfric_apps.git", "ref": "123"}, + {"source": "https://github.com/MetOffice/lfric_apps.git", "ref": "456"}, + ], + "repo3": [ + {"source": "https://github.com/MetOffice/lfric_core.git", "ref": "123"} + ], + "repo4": [{"source": "hostname:/path/to/repository", "ref": "123"}], + } + + assert set_https(input_dict) == output_dict diff --git a/github_scripts/tests/test_git_bdiff.py b/github_scripts/tests/test_git_bdiff.py index 27302f86..c261cea6 100644 --- a/github_scripts/tests/test_git_bdiff.py +++ b/github_scripts/tests/test_git_bdiff.py @@ -27,7 +27,16 @@ def add_to_repo(start, end, message, mode="wt"): print(f"Lorem ipsum dolor sit amet {i}", file=fd) subprocess.run(["git", "add", "-A"], check=True) - subprocess.run(["git", "commit", "--no-gpg-sign", "-m", message], check=True) + subprocess.run( + [ + "git", + "commit", + "--no-gpg-sign", + "-m", + message, + ], + check=True, + ) @pytest.fixture(scope="session") @@ -38,7 +47,7 @@ def git_repo(tmpdir_factory): os.chdir(location) # Create the repo and add some files - subprocess.run(["git", "init"], check=True) + subprocess.run(["git", "init", "--initial-branch=main"], check=True) add_to_repo(0, 10, "Testing") # Create a branch and add some files diff --git a/lfric_macros/apply_macros.py b/lfric_macros/apply_macros.py index 2a48ada7..4047ce5f 100755 --- a/lfric_macros/apply_macros.py +++ b/lfric_macros/apply_macros.py @@ -235,7 +235,7 @@ def read_python_imports(path): def banner_print(message): """Print a simple banner message""" - print(f"\n{(len(message)+4)*'*'}\n* {message} *\n{(len(message)+4)*'*'}\n") + print(f"\n{(len(message) + 4) * '*'}\n* {message} *\n{(len(message) + 4) * '*'}\n") class ApplyMacros: @@ -243,7 +243,7 @@ class ApplyMacros: Object to hold data + methods to apply upgrade macros in lfric_apps """ - def __init__(self, tag, cname, version, apps, core, jules): + def __init__(self, tag, cname, version, apps, core, jules, testing=False): self.tag = tag if cname: self.class_name = cname @@ -252,13 +252,16 @@ def __init__(self, tag, cname, version, apps, core, jules): # removed from the version self.class_name = tag.replace(".", "") self.temp_dirs = {} - self.root_path = get_root_path(apps) + if testing: + # Don't search for a git repo if testing + self.root_path = apps + else: + self.root_path = get_root_path(apps) self.core_source = self.get_dependency_paths(core, "lfric_core") # The Jules source is temporarily ignored as Jules Shared metadata has a # copy in LFRic, rather than using the Jules version. The LFRic build # system needs modifying to enable this # self.jules_source = self.get_dependency_paths(jules, "jules") - self.central_rose_meta = False self.set_rose_meta_path() if version is None: self.version = re.search(r".*vn(\d+\.\d+)(_.*)?", tag).group(1) @@ -278,20 +281,16 @@ def __init__(self, tag, cname, version, apps, core, jules): def set_rose_meta_path(self): """ Set up the ROSE_META_PATH environment variable in order to use the Jules - and Core metadata. We also add the working copy root path as this should + and Core metadata. We also add the clone root path as this should allow the script to be run from anywhere. When Jules Shared from Jules is enabled, self.jules_source will need adding here + Edit 02/2026 - remove backwards compatibility support for pre central-metadata """ - if os.path.isdir(os.path.join(self.root_path, "rose-meta")): - # For backwards compatibility with central rose-meta imports - rose_meta_path = ( - f"{os.path.join(self.root_path, 'rose-meta')}:" - f"{os.path.join(self.core_source, 'rose-meta')}" - ) - self.central_rose_meta = True - else: - rose_meta_path = f"{self.root_path}:{self.core_source}" + rose_meta_path = ( + f"{os.path.join(self.root_path, 'rose-meta')}:" + f"{os.path.join(self.core_source, 'rose-meta')}" + ) os.environ["ROSE_META_PATH"] = rose_meta_path def parse_application_section(self, meta_dir): @@ -413,10 +412,12 @@ def find_meta_dirs(self, path): - str, stdout of find command looking for versions.py files """ + meta_dirs = set() for dirpath, dirnames, filenames in os.walk(path, followlinks=True): - dirnames[:] = [d for d in dirnames if d not in [".svn"]] + dirnames[:] = [d for d in dirnames if d not in [".svn", ".git"]] if "versions.py" in filenames: - self.meta_dirs.add(dirpath) + meta_dirs.add(dirpath) + return meta_dirs def parse_macro(self, macro, meta_dir): """ @@ -442,17 +443,16 @@ def parse_macro(self, macro, meta_dir): author = "Unknown" pass - class_name = re.search(r"class (vn\d+_t\d+)", macro).group(1) - # Search for the before tag # Raise an exception if these are missing try: before_tag = re.search(rf"BEFORE_TAG{TAG_REGEX}", macro).group(1) after_tag = re.search(rf"AFTER_TAG{TAG_REGEX}", macro).group(1) + class_name = re.search(r"class (vn\d+_t\d+)", macro).group(1) except AttributeError as exc: raise Exception( - "Couldn't find a Before/After tag for the requested " - f"macro in the file {version_file}" + "Couldn't find a Before/After tag or class name for the macro:\n\n " + f"{macro}\n\nin the file {version_file}" ) from exc # Finally extract the lines which are defining the commands run by the @@ -592,13 +592,8 @@ def get_full_import_path(self, imp): # TODO: Reinstate Jules checks when using Jules Metadata from Jules - # For backwards compatibility with central rose-meta imports - if self.central_rose_meta: - core_imp = os.path.join(self.core_source, "rose-meta", imp) - apps_imp = os.path.join(self.root_path, "rose-meta", imp) - else: - core_imp = os.path.join(self.core_source, imp) - apps_imp = os.path.join(self.root_path, imp) + core_imp = os.path.join(self.core_source, "rose-meta", imp) + apps_imp = os.path.join(self.root_path, "rose-meta", imp) if os.path.exists(core_imp): return core_imp @@ -707,11 +702,7 @@ def determine_import_order(self, app): - A list of meta imports in the correct order """ - # If using central metadata, use the basename, otherwise the full path - if self.central_rose_meta: - app_name = os.path.basename(app) - else: - app_name = app + app_name = os.path.basename(app) import_list = [app_name] @@ -785,7 +776,7 @@ def write_new_macro(self, meta_dir, full_command, macro): with open(temppath, "a") as f: f.write( - f'class {macro["class_name"]}(MacroUpgrade):\n' + f"class {macro['class_name']}(MacroUpgrade):\n" f' """Upgrade macro for ticket {ticket_number} ' f'by {author}."""\n\n' f' BEFORE_TAG = "{macro["before_tag"]}"\n' @@ -945,13 +936,12 @@ def preprocess_macros(self): """ # Get list of versions files to check - in both core and apps - # Duplicated for backwards compatibility with central rose-meta imports - if self.central_rose_meta: + self.meta_dirs = self.meta_dirs.union( self.find_meta_dirs(os.path.join(self.root_path, "rose-meta")) + ) + self.meta_dirs = self.meta_dirs.union( self.find_meta_dirs(os.path.join(self.core_source, "rose-meta")) - else: - self.find_meta_dirs(self.root_path) - self.find_meta_dirs(self.core_source) + ) for meta_dir in self.meta_dirs: print( @@ -1052,6 +1042,20 @@ def metadata_check(self, meta_dir): ) print(f"[PASS] {meta_dir} validated") + def get_rose_apps(self): + """ + Return: + - list of paths to rose-stem apps in Apps, Core and Jules + """ + + apps_list = [] + for item in (self.root_path, self.core_source): + app_dir = os.path.join(item, "rose-stem", "app") + if not os.path.exists(app_dir): + continue + apps_list += [os.path.join(app_dir, f) for f in os.listdir(app_dir)] + return set(apps_list) + def apps_to_upgrade(self): """ Loop over rose-stem apps, finding ones using metadata with an upgrade @@ -1060,10 +1064,8 @@ def apps_to_upgrade(self): - list of paths to apps requiring upgrading """ upgradeable_apps = [] - app_dir_apps = os.path.join(self.root_path, "rose-stem", "app") - app_dir_core = os.path.join(self.core_source, "rose-stem", "app") - apps_list = [os.path.join(app_dir_apps, f) for f in os.listdir(app_dir_apps)] - apps_list += [os.path.join(app_dir_core, f) for f in os.listdir(app_dir_core)] + apps_list = self.get_rose_apps() + for app_path in apps_list: # Ignore lfric_coupled_rivers as this is based on Jules-standalone # metadata which is not currently available diff --git a/lfric_macros/check_macro_chains.py b/lfric_macros/check_macro_chains.py index f606e30b..df834837 100755 --- a/lfric_macros/check_macro_chains.py +++ b/lfric_macros/check_macro_chains.py @@ -130,7 +130,6 @@ def main(): errors = [] for meta_dir in macro_object.meta_dirs: - before_tags = find_macro_tags("before", meta_dir, errors) after_tags = find_macro_tags("after", meta_dir, errors) diff --git a/lfric_macros/release_lfric.py b/lfric_macros/release_lfric.py index 74df2df6..a286fc4f 100755 --- a/lfric_macros/release_lfric.py +++ b/lfric_macros/release_lfric.py @@ -415,7 +415,6 @@ def parse_args(): def main(): - args = parse_args() macro_object = ApplyMacros( diff --git a/lfric_macros/tests/test_apply_macros.py b/lfric_macros/tests/test_apply_macros.py index e6aeb11d..c6928d4f 100644 --- a/lfric_macros/tests/test_apply_macros.py +++ b/lfric_macros/tests/test_apply_macros.py @@ -1,9 +1,31 @@ -import shutil -import subprocess - +# ----------------------------------------------------------------------------- +# (C) Crown copyright Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +# ----------------------------------------------------------------------------- +""" +Unit tests for apply_macros +""" + +from os import path import pytest -from ..apply_macros import * # noqa: F403 +from ..apply_macros import ( + ApplyMacros, + split_macros, + match_python_import, + deduplicate_list, + read_versions_file, + read_python_imports, +) + +# Commonly used paths +TEST_APPS_DIR = path.join( + path.dirname(path.dirname(path.abspath(__file__))), "tests", "test_lfric_apps_dir" +) +TEST_META_DIR = path.join(TEST_APPS_DIR, "rose-meta") +TEST_ROSE_STEM = path.join(TEST_APPS_DIR, "rose-stem", "app") + # A macro that we want to find for these tests desired_macro = """class vn00_t001(MacroUpgrade): @@ -20,7 +42,7 @@ def upgrade(self, config, meta_config=None): # A macro pre-existing in the file, forming part of the chain existing_macro = """class vn00_t000(MacroUpgrade): - # Upgrade macro for ticket #000 by . + # Upgrade macro for ticket #000 by . BEFORE_TAG = "vn0.0" AFTER_TAG = "vn0.0_t000" def upgrade(self, config, meta_config=None): @@ -48,138 +70,294 @@ def __repr__(self): # The expected result from split_macros for the versions expected_split_macros = [desired_macro, existing_macro] -# ApplyMacros below requires an LFRic Apps working copy to work - check out the -# head of the lfric_apps trunk for this purpose. The actual contents of the -# working copy are not important for the purposes of the unit tests -appsdir = tempfile.mkdtemp() -result = subprocess.run( - f"fcm co fcm:lfric_apps.x_tr {appsdir}".split(), - check=False, - capture_output=True, - text=True, - timeout=120, -) -if result.returncode: - raise RuntimeError( - "Failed to checkout required LFRic Apps Working Copy with error ", - result.stderr, - ) - # Create an instance of the apply_macros class -# Pass a known directory in as the Jules and Core sources as these are not -# required for testing -am = ApplyMacros("vn0.0_t001", None, None, appsdir, "/tmp", "/tmp") +# Use /tmp for Core and Jules as these are not required for testing +applymacros = ApplyMacros("vn0.0_t001", None, None, TEST_APPS_DIR, "/tmp", "/tmp", True) + + +def test_read_versions_file(): + """ + Test read_versions_file + """ + + assert read_versions_file( + path.join(TEST_APPS_DIR, "rose-meta", "lfric-gungho") + ) == ["# line 1\n", "# line 2\n"] + + +def test_find_meta_dirs(): + result = applymacros.find_meta_dirs(path.join(TEST_APPS_DIR, "rose-meta")) + expected = set() + expected.add(path.join(TEST_META_DIR, "lfric-lfric_atm")) + expected.add(path.join(TEST_META_DIR, "lfric-gungho")) + expected.add(path.join(TEST_META_DIR, "lfric-driver")) + expected.add(path.join(TEST_META_DIR, "um-iau")) + expected.add(path.join(TEST_META_DIR, "lfric-transport")) + assert result == expected + applymacros.meta_dirs = result def test_split_macros(): - m = split_macros(test_versions_file) + result = split_macros(test_versions_file) # Remove trailing newlines as these are unimportant - for i, item in enumerate(m): - m[i] = item.strip("\n") - assert m == expected_split_macros + for i, item in enumerate(result): + result[i] = item.strip("\n") + assert result == expected_split_macros + + +def test_match_python_imports(): + assert match_python_import("import z") + assert match_python_import("from x import y") + assert match_python_import("from a import b.c") + assert match_python_import("import m as n") + assert not match_python_import("false") + + +def test_deduplicate_list(): + """ + Test deduplicate_list + """ + + before = [1, 2, 2, "a", "b", "b", {0: 1, 1: 2}, {0: 1, 1: 2}] + after = [ + 1, + 2, + "a", + "b", + {0: 1, 1: 2}, + ] + + assert deduplicate_list(before) == after + assert deduplicate_list(after) == after + + +def test_read_python_imports(): + """ + Test read_python_imports + """ + + test_file = path.join(TEST_APPS_DIR, "test_read_python_imports.py") + expected = set() + imports = ( + (("shlex",), ("split",), None), + (("pathlib",), ("PurePath",), None), + ((), ("os",), None), + (("pathlib",), ("Path",), None), + ((), ("networkx",), "nx"), + ) + for item in imports: + expected.add(item) + assert read_python_imports(test_file) == expected def test_find_macro(): - assert am.find_macro("meta_dir", expected_split_macros) == desired_macro - assert am.find_macro("meta_dir", [existing_macro]) == "" + assert applymacros.find_macro("meta_dir", expected_split_macros) == desired_macro + assert applymacros.find_macro("meta_dir", [existing_macro]) == "" expected_error = r".*meta_dir/versions.py.*" with pytest.raises(Exception, match=expected_error): - am.find_macro("meta_dir", [""]) + applymacros.find_macro("meta_dir", [""]) def test_find_last_macro(): - assert am.find_last_macro(expected_split_macros, "meta_dir") == "vn0.0_t001" + assert ( + applymacros.find_last_macro(expected_split_macros, "meta_dir") == "vn0.0_t001" + ) def test_parse_macro(): - am.parse_macro(desired_macro, "meta_dir") - expected_dict = { - "before_tag": "vn0.0_t000", - "commands": ( - " self.add_setting(\n" - ' config, ["namelist:namelist1", "opt1"], "value1"\n' - " )\n" - ), - } - assert am.parsed_macros["meta_dir"] == expected_dict - assert am.ticket_number == "#001" - assert am.author == "Test Author" + for macro in (existing_macro, desired_macro): + applymacros.parsed_macros["meta_dir"].append( + applymacros.parse_macro(macro, "meta_dir") + ) + expected_macros_list = [ + { + "before_tag": "vn0.0", + "after_tag": "vn0.0_t000", + "commands": ( + " self.add_setting(\n" + ' config, ["namelist:namelist0", "opt0"], "value0"\n' + " )\n" + ), + "ticket_number": "#000", + "author": "Previous Author", + "class_name": "vn00_t000", + }, + { + "before_tag": "vn0.0_t000", + "after_tag": "vn0.0_t001", + "commands": ( + " self.add_setting(\n" + ' config, ["namelist:namelist1", "opt1"], "value1"\n' + " )\n" + ), + "ticket_number": "#001", + "author": "Test Author", + "class_name": "vn00_t001", + }, + ] + assert applymacros.parsed_macros["meta_dir"] == expected_macros_list with pytest.raises(Exception, match=r".*failed/versions.py"): - am.parse_macro("", "failed") + applymacros.parse_macro("", "failed") + + +def test_check_missing_macro(): + macros = applymacros.parsed_macros["meta_dir"] + applymacros.parsed_macros[path.join(TEST_META_DIR, "lfric-gungho")] = macros + missing = applymacros.check_missing_macros( + path.join(TEST_META_DIR, "lfric-lfric_atm"), ["lfric-gungho"] + ) + assert missing == ["vn0.0_t000"] + missing = applymacros.check_missing_macros( + path.join(TEST_META_DIR, "lfric-gungho"), [] + ) + assert missing == [] + + +def test_combine_missing_macros(): + combined = applymacros.combine_missing_macros( + [path.join(TEST_META_DIR, "lfric-gungho")], ["vn0.0_t000"] + ) + expected_combined = { + "vn0.0_t000": { + "before_tag": "vn0.0", + "after_tag": "vn0.0_t000", + "commands": ( + " self.add_setting(\n" + ' config, ["namelist:namelist0", "opt0"], "value0"\n' + " )\n" + ), + "ticket_number": "#000", + "author": "Previous Author", + "class_name": "vn00_t000", + }, + } + assert combined == expected_combined def test_read_meta_imports(): - am.parsed_macros["tests/test_meta_dir"] = {} - am.parsed_macros["tests/test_meta_dir"]["imports"] = am.read_meta_imports( - "tests/test_meta_dir" + applymacros.parsed_macros[TEST_APPS_DIR] = {} + applymacros.parsed_macros[TEST_APPS_DIR]["imports"] = applymacros.read_meta_imports( + path.join(TEST_APPS_DIR, "rose-meta", "lfric-gungho") ) expected_imports = [ - os.path.join(am.root_path, "science", "gungho"), - os.path.join(am.root_path, "applications", "lfric_atm"), + path.join(applymacros.root_path, "rose-meta", "lfric-driver"), + path.join(applymacros.root_path, "rose-meta", "um-iau"), + ] + assert applymacros.parsed_macros[TEST_APPS_DIR]["imports"] == expected_imports + + expected_meta = [ + path.join(applymacros.root_path, "rose-meta", "lfric-lfric_atm", "vn0.0") ] - assert am.parsed_macros["tests/test_meta_dir"]["imports"] == expected_imports - expected_meta = [os.path.join(am.root_path, "applications", "lfric_atm")] assert ( - am.read_meta_imports("tests/test_meta_dir/rose-app.conf", "meta") + applymacros.read_meta_imports( + path.join(TEST_APPS_DIR, "rose-stem", "app", "lfric_atm", "rose-app.conf"), + "meta", + ) == expected_meta ) def test_determine_import_order(): - am.parsed_macros["import1"] = {} - am.parsed_macros["import2"] = {} - am.parsed_macros["import3"] = {} - am.parsed_macros["import4"] = {} - am.parsed_macros["import1"]["imports"] = ["import3", "import2"] - am.parsed_macros["import3"]["imports"] = ["import4"] - am.parsed_macros["import4"]["imports"] = [] - am.parsed_macros["import2"]["imports"] = [] + applymacros.target_macros["import1"] = {} + applymacros.target_macros["import2"] = {} + applymacros.target_macros["import3"] = {} + applymacros.target_macros["import4"] = {} + applymacros.target_macros["import1"]["imports"] = ["import3", "import2"] + applymacros.target_macros["import3"]["imports"] = ["import4"] + applymacros.target_macros["import4"]["imports"] = [] + applymacros.target_macros["import2"]["imports"] = [] expected_order = ["import2", "import4", "import3", "import1"] - assert am.determine_import_order("import1") == expected_order + assert applymacros.determine_import_order("import1") == expected_order def test_combine_macros(): - am.parsed_macros["importA"] = {"commands": " importA command"} - am.parsed_macros["importB"] = {"commands": " importB command"} + applymacros.target_macros[applymacros.get_full_import_path("importA")] = { + "commands": " importA command", + "author": "Test Author", + "ticket_number": "#001", + } + applymacros.target_macros[applymacros.get_full_import_path("importB")] = { + "commands": " importB command", + "author": "Test Author", + "ticket_number": "#001", + } expected_combined = ( - " # Commands From: importA\n importA command\n" - " # Commands From: importB\n importB command\n" + " # Commands From: rose-meta/importA\n importA command\n" + " # Commands From: rose-meta/importB\n importB command\n" ) - assert am.combine_macros(["importA", "importB"]) == expected_combined + result = applymacros.combine_macros(["importA", "importB"]) + assert result == expected_combined def test_parse_application_section(): - assert am.parse_application_section("meta_dir/HEAD") == "meta_dir" - assert am.parse_application_section("meta_dir/versions.py") == "meta_dir" - assert am.parse_application_section(f"{am.root_path}/meta_dir") == "meta_dir" - assert am.parse_application_section(f"{am.core_source}/meta_dir") == "meta_dir" - - -def test_deduplicate_list(): - assert deduplicate_list([1, 2, 3]) == [1, 2, 3] - assert deduplicate_list( - [ - 1, - 2, - 2, - 3, - 3, - 3, - ] - ) == [1, 2, 3] - assert deduplicate_list([1, 2, 1, 3, 2]) == [1, 2, 3] - - -def test_match_python_imports(): - assert match_python_import("import z") - assert match_python_import("from x import y") - assert match_python_import("from a import b.c") - assert match_python_import("import m as n") - assert not match_python_import("false") + assert ( + applymacros.parse_application_section(path.join("meta_dir", "HEAD")) + == "meta_dir" + ) + assert ( + applymacros.parse_application_section(path.join("meta_dir", "versions.py")) + == "meta_dir" + ) + assert ( + applymacros.parse_application_section( + path.join(applymacros.root_path, "meta_dir") + ) + == "meta_dir" + ) + assert ( + applymacros.parse_application_section( + path.join(applymacros.core_source, "meta_dir") + ) + == "meta_dir" + ) -# Remove appsdir -@pytest.fixture(scope="session", autouse=True) -def remove_tempdir(): - yield - shutil.rmtree(appsdir) +def test_read_dependencies(): + assert applymacros.read_dependencies("test_repo") == ("test_source", "test_ref") + + +def test_order_meta_dirs(): + applymacros.target_macros = { + path.join(TEST_META_DIR, "lfric-driver"): {"imports": []}, + path.join(TEST_META_DIR, "lfric-gungho"): { + "imports": [ + path.join(TEST_META_DIR, "lfric-driver"), + path.join(TEST_META_DIR, "um-iau"), + ] + }, + path.join(TEST_META_DIR, "um-iau"): {"imports": []}, + path.join(TEST_META_DIR, "lfric-lfric_atm"): { + "imports": [path.join(TEST_META_DIR, "lfric-gungho")] + }, + path.join(TEST_META_DIR, "lfric-transport"): { + "imports": [path.join(TEST_META_DIR, "lfric-gungho")] + }, + } + order = applymacros.order_meta_dirs() + gungho = order.index(path.join(TEST_META_DIR, "lfric-gungho")) + lfric_atm = order.index(path.join(TEST_META_DIR, "lfric-lfric_atm")) + driver = order.index(path.join(TEST_META_DIR, "lfric-driver")) + um = order.index(path.join(TEST_META_DIR, "um-iau")) + assert gungho > driver + assert gungho > um + assert lfric_atm > gungho + + +def test_get_rose_apps(): + expected = set() + expected.add(path.join(TEST_ROSE_STEM, "gungho")) + expected.add(path.join(TEST_ROSE_STEM, "lfric_atm")) + expected.add(path.join(TEST_ROSE_STEM, "transport")) + assert applymacros.get_rose_apps() == expected + + +def test_apps_to_upgrade(): + applymacros.sections_with_macro = [ + path.join(TEST_META_DIR, "lfric-gungho"), + path.join(TEST_META_DIR, "lfric-lfric_atm"), + ] + expected = ( + [path.join(TEST_ROSE_STEM, "gungho"), path.join(TEST_ROSE_STEM, "lfric_atm")], + [path.join(TEST_ROSE_STEM, "lfric_atm"), path.join(TEST_ROSE_STEM, "gungho")], + ) + assert applymacros.apps_to_upgrade() in expected diff --git a/lfric_macros/tests/test_lfric_apps_dir/dependencies.yaml b/lfric_macros/tests/test_lfric_apps_dir/dependencies.yaml new file mode 100644 index 00000000..8e637731 --- /dev/null +++ b/lfric_macros/tests/test_lfric_apps_dir/dependencies.yaml @@ -0,0 +1,3 @@ +test_repo: + source: test_source + ref: test_ref diff --git a/lfric_macros/tests/test_lfric_apps_dir/rose-meta/lfric-driver/HEAD/rose-meta.conf b/lfric_macros/tests/test_lfric_apps_dir/rose-meta/lfric-driver/HEAD/rose-meta.conf new file mode 100644 index 00000000..e69de29b diff --git a/lfric_macros/tests/test_lfric_apps_dir/rose-meta/lfric-driver/versions.py b/lfric_macros/tests/test_lfric_apps_dir/rose-meta/lfric-driver/versions.py new file mode 100644 index 00000000..e69de29b diff --git a/lfric_macros/tests/test_lfric_apps_dir/rose-meta/lfric-gungho/HEAD/rose-meta.conf b/lfric_macros/tests/test_lfric_apps_dir/rose-meta/lfric-gungho/HEAD/rose-meta.conf new file mode 100644 index 00000000..e2bcd8da --- /dev/null +++ b/lfric_macros/tests/test_lfric_apps_dir/rose-meta/lfric-gungho/HEAD/rose-meta.conf @@ -0,0 +1,2 @@ +import=lfric-driver/HEAD + =um-iau/HEAD diff --git a/lfric_macros/tests/test_lfric_apps_dir/rose-meta/lfric-gungho/versions.py b/lfric_macros/tests/test_lfric_apps_dir/rose-meta/lfric-gungho/versions.py new file mode 100644 index 00000000..d4fecb93 --- /dev/null +++ b/lfric_macros/tests/test_lfric_apps_dir/rose-meta/lfric-gungho/versions.py @@ -0,0 +1,3 @@ +# line 1 + +# line 2 diff --git a/lfric_macros/tests/test_lfric_apps_dir/rose-meta/lfric-lfric_atm/HEAD/rose-meta.conf b/lfric_macros/tests/test_lfric_apps_dir/rose-meta/lfric-lfric_atm/HEAD/rose-meta.conf new file mode 100644 index 00000000..d28a3e2f --- /dev/null +++ b/lfric_macros/tests/test_lfric_apps_dir/rose-meta/lfric-lfric_atm/HEAD/rose-meta.conf @@ -0,0 +1 @@ +import=lfric-gungho/HEAD diff --git a/lfric_macros/tests/test_lfric_apps_dir/rose-meta/lfric-lfric_atm/versions.py b/lfric_macros/tests/test_lfric_apps_dir/rose-meta/lfric-lfric_atm/versions.py new file mode 100644 index 00000000..e69de29b diff --git a/lfric_macros/tests/test_lfric_apps_dir/rose-meta/lfric-transport/HEAD/rose-meta.conf b/lfric_macros/tests/test_lfric_apps_dir/rose-meta/lfric-transport/HEAD/rose-meta.conf new file mode 100644 index 00000000..d28a3e2f --- /dev/null +++ b/lfric_macros/tests/test_lfric_apps_dir/rose-meta/lfric-transport/HEAD/rose-meta.conf @@ -0,0 +1 @@ +import=lfric-gungho/HEAD diff --git a/lfric_macros/tests/test_lfric_apps_dir/rose-meta/lfric-transport/versions.py b/lfric_macros/tests/test_lfric_apps_dir/rose-meta/lfric-transport/versions.py new file mode 100644 index 00000000..e69de29b diff --git a/lfric_macros/tests/test_lfric_apps_dir/rose-meta/um-iau/HEAD/rose-meta.conf b/lfric_macros/tests/test_lfric_apps_dir/rose-meta/um-iau/HEAD/rose-meta.conf new file mode 100644 index 00000000..e69de29b diff --git a/lfric_macros/tests/test_lfric_apps_dir/rose-meta/um-iau/versions.py b/lfric_macros/tests/test_lfric_apps_dir/rose-meta/um-iau/versions.py new file mode 100644 index 00000000..e69de29b diff --git a/lfric_macros/tests/test_lfric_apps_dir/rose-stem/app/gungho/rose-app.conf b/lfric_macros/tests/test_lfric_apps_dir/rose-stem/app/gungho/rose-app.conf new file mode 100644 index 00000000..fcffc340 --- /dev/null +++ b/lfric_macros/tests/test_lfric_apps_dir/rose-stem/app/gungho/rose-app.conf @@ -0,0 +1,3 @@ +meta=lfric-gungho/vn0.0 + +[metadata] diff --git a/lfric_macros/tests/test_lfric_apps_dir/rose-stem/app/lfric_atm/rose-app.conf b/lfric_macros/tests/test_lfric_apps_dir/rose-stem/app/lfric_atm/rose-app.conf new file mode 100644 index 00000000..7fe8cf54 --- /dev/null +++ b/lfric_macros/tests/test_lfric_apps_dir/rose-stem/app/lfric_atm/rose-app.conf @@ -0,0 +1,3 @@ +meta=lfric-lfric_atm/vn0.0 + +[metadata] diff --git a/lfric_macros/tests/test_lfric_apps_dir/rose-stem/app/transport/rose-app.conf b/lfric_macros/tests/test_lfric_apps_dir/rose-stem/app/transport/rose-app.conf new file mode 100644 index 00000000..e9a207ee --- /dev/null +++ b/lfric_macros/tests/test_lfric_apps_dir/rose-stem/app/transport/rose-app.conf @@ -0,0 +1,3 @@ +meta=lfric-transport/vn0.0 + +[metadata] diff --git a/lfric_macros/tests/test_lfric_apps_dir/test_read_python_imports.py b/lfric_macros/tests/test_lfric_apps_dir/test_read_python_imports.py new file mode 100644 index 00000000..bc1333d0 --- /dev/null +++ b/lfric_macros/tests/test_lfric_apps_dir/test_read_python_imports.py @@ -0,0 +1,4 @@ +import os +from shlex import split +from pathlib import Path, PurePath +import networkx as nx diff --git a/lfric_macros/tests/test_meta_dir/HEAD/rose-meta.conf b/lfric_macros/tests/test_meta_dir/HEAD/rose-meta.conf deleted file mode 100644 index 16b43570..00000000 --- a/lfric_macros/tests/test_meta_dir/HEAD/rose-meta.conf +++ /dev/null @@ -1,4 +0,0 @@ -import=science/gungho - =applications/lfric_atm - -[metadata] diff --git a/lfric_macros/tests/test_meta_dir/rose-app.conf b/lfric_macros/tests/test_meta_dir/rose-app.conf deleted file mode 100644 index 4aa90c52..00000000 --- a/lfric_macros/tests/test_meta_dir/rose-app.conf +++ /dev/null @@ -1,3 +0,0 @@ -meta=applications/lfric_atm - -[metadata] diff --git a/lfric_macros/validate_rose_meta.py b/lfric_macros/validate_rose_meta.py index c94301e6..7ea21f70 100755 --- a/lfric_macros/validate_rose_meta.py +++ b/lfric_macros/validate_rose_meta.py @@ -217,17 +217,17 @@ def main(): rose_meta_path = "" if args.apps: source_path = args.apps - meta_paths += f"-M {os.path.join(args.apps, "rose-meta")} " - rose_meta_path += f"{os.path.join(args.apps, "rose-meta")}" + meta_paths += f"-M {os.path.join(args.apps, 'rose-meta')} " + rose_meta_path += f"{os.path.join(args.apps, 'rose-meta')}" if args.core: - meta_paths += f"-M {os.path.join(args.core, "rose-meta")} " + meta_paths += f"-M {os.path.join(args.core, 'rose-meta')} " if rose_meta_path: # Apps has already started this - rose_meta_path += f":{os.path.join(args.core, "rose-meta")}" + rose_meta_path += f":{os.path.join(args.core, 'rose-meta')}" else: # Apps hasn't been set source_path = args.core - rose_meta_path = f"{os.path.join(args.core, "rose-meta")}" + rose_meta_path = f"{os.path.join(args.core, 'rose-meta')}" if check_rose_metadata(rose_meta_path, source_path) or check_rose_stem_apps( meta_paths, source_path diff --git a/nightly_testing/__init__.py b/nightly_testing/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nightly_testing/generate_test_suite_cron.py b/nightly_testing/generate_test_suite_cron.py index e5f77f67..e0f61028 100755 --- a/nightly_testing/generate_test_suite_cron.py +++ b/nightly_testing/generate_test_suite_cron.py @@ -53,9 +53,9 @@ "ukca": [], } -CLONE_DIR = os.path.join(os.environ["TMPDIR"], os.environ["USER"]) +CLONE_DIR = os.path.join(os.getenv("TMPDIR", "."), os.getenv("USER", "unknown")) MIRROR_PATH = "/data/users/gitassist/git_mirrors/" -UMDIR = os.environ["UMDIR"] +UMDIR = os.getenv("UMDIR", ".") CYLC = "bash -l cylc" DATE_BASE = "date +\\%Y-\\%m-\\%d" @@ -80,7 +80,7 @@ def create_git_clone_cron(repo): command = f"# Clone {repo} - every day at 23:30 #" length = len(command) - command = f"{length*'#'}\n{command}\n{length*'#'}\n30 23 * * * " + command = f"{length * '#'}\n{command}\n{length * '#'}\n30 23 * * * " command += f"rm -rf {clone_path} ; " command += f"git clone {repo_mirror} {clone_path}" return command + "\n\n\n" @@ -172,7 +172,7 @@ def generate_clean_cron(suite_name, suite, log_file, cylc_version): return clean_cron -def generate_cylc_command(suite, wc_path, cylc_version, name): +def generate_cylc_command(suite, clone_path, cylc_version, name): """ Return a string with the rose-stem command Ignores any additional source arguments @@ -186,7 +186,7 @@ def generate_cylc_command(suite, wc_path, cylc_version, name): ) if "revisions" in suite and suite["revisions"] == "heads": command += "-S USE_HEADS=true " - command += f"{os.path.join(wc_path, 'rose-stem')} " + command += f"{os.path.join(clone_path, 'rose-stem')} " return command diff --git a/nightly_testing/retrigger_nightlies.py b/nightly_testing/retrigger_nightlies.py index 387edfb6..adb72f4a 100755 --- a/nightly_testing/retrigger_nightlies.py +++ b/nightly_testing/retrigger_nightlies.py @@ -51,8 +51,7 @@ def check_for_workflow_params(conn): This is used as a proxy for whether a suite is cylc8 - if return True, it is """ res = conn.execute( - "SELECT name FROM sqlite_master " - "WHERE type='table' AND name='workflow_params'" + "SELECT name FROM sqlite_master WHERE type='table' AND name='workflow_params'" ).fetchall() if len(res) > 0: return True @@ -102,7 +101,7 @@ def retrigger_suite(suite, tasks): print(f"\nTriggering Failed Tasks in {suite}") ntasks = len(tasks) for i, task in enumerate(tasks): - print(f"\rTask {i+1}/{ntasks}", end="", flush=True) + print(f"\rTask {i + 1}/{ntasks}", end="", flush=True) failed_command = f"cylc trigger {suite}//*/{task[0]}" if "next-cylc" in suite: failed_command = f"export CYLC_VERSION=8-next ; {failed_command}" diff --git a/nightly_testing/tests/__init__.py b/nightly_testing/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nightly_testing/tests/test_generate_test_suite_cron.py b/nightly_testing/tests/test_generate_test_suite_cron.py index d627c85a..6c287e9f 100644 --- a/nightly_testing/tests/test_generate_test_suite_cron.py +++ b/nightly_testing/tests/test_generate_test_suite_cron.py @@ -1,45 +1,13 @@ import pytest -from generate_test_suite_cron import * # noqa: F403 - -PROFILE = ". /etc/profile" - -# Test join_checkout_commands -data_join_checkout_commands = [ - ( - ["um"], - "scratch/dir/", - "fcm co -q --force fcm:um.xm_tr@HEAD scratch/dir/wc_um ; ", - ), - ( - ["um", "lfric"], - "scratch/dir", - "fcm co -q --force fcm:um.xm_tr@HEAD scratch/dir/wc_um ; fcm co -q --force fcm:lfric.xm_tr@HEAD scratch/dir/wc_lfric ; ", - ), -] - - -@pytest.mark.parametrize( - ("inlist", "scratch", "expected"), - [test_data for test_data in data_join_checkout_commands], +from ..generate_test_suite_cron import ( + MIRROR_PATH, + CLONE_DIR, + generate_cron_timing_str, + generate_clean_commands, + populate_cl_variables, + create_git_clone_cron, + generate_cylc_command, ) -def test_join_checkout_commands(inlist, scratch, expected): - assert join_checkout_commands(inlist, scratch) == expected - - -# Test lfric_heads_sed -data_lfric_heads_sed = [ - ( - "path/to/wc", - "cp -rf path/to/wc path/to/wc_heads ; sed -i -e 's/^\\(export .*_revision=@\\).*/\\1HEAD/' path/to/wc_heads/dependencies.sh ; sed -i -e 's/^\\(export .*_rev=\\).*/\\1HEAD/' path/to/wc_heads/dependencies.sh ; ", - ) -] - - -@pytest.mark.parametrize( - ("wc_path", "expected"), [test_data for test_data in data_lfric_heads_sed] -) -def test_lfric_heads_sed(wc_path, expected): - assert lfric_heads_sed(wc_path) == (expected) # Test generate_cron_timing_str @@ -58,17 +26,6 @@ def test_lfric_heads_sed(wc_path, expected): "clean", "30 00 * * 2-6 ", ), - ({"period": "weekly", "cron_clean": "30 00"}, "monitoring", "00 06 * * 1 "), - ( - {"period": "nightly", "cron_clean": "30 00"}, - "monitoring", - "00 06 * * 2-5 ", - ), - ( - {"period": "nightly_all", "cron_clean": "30 00"}, - "monitoring", - "00 06 * * 1-5 ", - ), ] @@ -82,23 +39,17 @@ def test_generate_cron_timing_str(suite, mode, expected): # Test generate_clean_commands data_generate_clean_commands = [ - ( - "7", - "suite_name", - "cron_log", - f"{PROFILE} ; export CYLC_VERSION=7 ; cylc stop 'suite_name' >/dev/null 2>&1 ; sleep 10 ; rose suite-clean -y -q suite_name >> cron_log 2>&1\n", - ), ( "8", "suite_name", "cron_log", - f"{PROFILE} ; export CYLC_VERSION=8 ; cylc stop 'suite_name' >/dev/null 2>&1 ; sleep 10 ; cylc clean --timeout=7200 -y -q suite_name >> cron_log 2>&1\n", + "export CYLC_VERSION=8 ; bash -l cylc stop --kill 'suite_name' >/dev/null 2>&1 ; sleep 10 ; bash -l cylc clean --timeout=7200 -y -q suite_name >> cron_log 2>&1\n", ), ( "8-next", "suite_name", "cron_log", - f"{PROFILE} ; export CYLC_VERSION=8-next ; cylc stop 'suite_name' >/dev/null 2>&1 ; sleep 10 ; cylc clean --timeout=7200 -y -q suite_name >> cron_log 2>&1\n", + "export CYLC_VERSION=8-next ; bash -l cylc stop --kill 'suite_name' >/dev/null 2>&1 ; sleep 10 ; bash -l cylc clean --timeout=7200 -y -q suite_name >> cron_log 2>&1\n", ), ] @@ -111,81 +62,6 @@ def test_generate_clean_commands(cylc_version, name, log_file, expected): assert generate_clean_commands(cylc_version, name, log_file) == expected -# Test generate_rose_stem_command -data_generate_rose_stem_command = [ - ( - {"groups": "all"}, - "path/to/wc", - "7", - "suite_name", - "export CYLC_VERSION=7 ; rose stem --group=all --name=suite_name --source=path/to/wc ", - ), - ( - {"groups": "nightly"}, - "path/to/wc", - "8", - "suite_name", - "export CYLC_VERSION=8 ; rose stem --group=nightly --workflow-name=suite_name --source=path/to/wc ", - ), - ( - {"groups": "nightly"}, - "path/to/wc", - "8-next", - "suite_name", - "export CYLC_VERSION=8-next ; rose stem --group=nightly --workflow-name=suite_name --source=path/to/wc ", - ), -] - - -@pytest.mark.parametrize( - ("suite", "wc_path", "cylc_version", "name", "expected"), - [test_data for test_data in data_generate_rose_stem_command], -) -def test_generate_rose_stem_command(suite, wc_path, cylc_version, name, expected): - assert generate_rose_stem_command(suite, wc_path, cylc_version, name) == expected - - -# Test populate_heads_sources -data_populate_heads_sources = [ - ( - { - "repo": "um", - "revisions": "heads", - }, - "--source=fcm:casim.xm_tr@HEAD --source=fcm:jules.xm_tr@HEAD --source=fcm:mule.xm_tr@HEAD --source=fcm:shumlib.xm_tr@HEAD --source=fcm:socrates.xm_tr@HEAD --source=fcm:ukca.xm_tr@HEAD ", - ), - ( - { - "repo": "um", - "revisions": "set", - }, - "", - ), - ( - { - "repo": "um", - }, - "", - ), - ( - { - "repo": "jules", - "revisions": "heads", - }, - "", - ), - ({"repo": "lfric_apps", "revisions": "heads"}, ""), -] - - -@pytest.mark.parametrize( - ("suite", "expected"), - [test_data for test_data in data_populate_heads_sources], -) -def test_populate_heads_sources(suite, expected): - assert populate_heads_sources(suite) == expected - - # Test populate_cl_variables data_populate_cl_variables = [ ({"vars": ["var1", "var2", "var3"]}, "-S var1 -S var2 -S var3 "), @@ -201,17 +77,53 @@ def test_populate_cl_variables(suite, expected): assert populate_cl_variables(suite) == expected -# Test major_cylc_version -data_major_cylc_version = [ - ("7", "7"), - ("8", "8"), - ("8-next", "8"), +# Test create_git_clone_cron +def test_create_git_clone_cron(): + expected = ( + "###################################\n" + "# Clone repo - every day at 23:30 #\n" + "###################################\n" + f"30 23 * * * rm -rf {CLONE_DIR}/clone_repo ; git clone {MIRROR_PATH}MetOffice/repo.git {CLONE_DIR}/clone_repo\n\n\n" + ) + assert create_git_clone_cron("repo") == expected + + +# Test generate_cylc_command +data_generate_cylc_command = [ + ( + {"groups": "groups"}, + "/path/to/clone", + "8", + "test_name", + "export CYLC_VERSION=8 ; bash -l cylc vip -z g=groups -n test_name -S USE_MIRRORS=true /path/to/clone/rose-stem ", + ), + ( + {"groups": "groups", "revisions": "set"}, + "/path/to/clone", + "8", + "test_name", + "export CYLC_VERSION=8 ; bash -l cylc vip -z g=groups -n test_name -S USE_MIRRORS=true /path/to/clone/rose-stem ", + ), + ( + {"groups": "groups", "revisions": "set"}, + "/path/to/clone", + "8-next", + "test_name", + "export CYLC_VERSION=8-next ; bash -l cylc vip -z g=groups -n test_name -S USE_MIRRORS=true /path/to/clone/rose-stem ", + ), + ( + {"groups": "groups", "revisions": "heads"}, + "/path/to/clone", + "8", + "test_name", + "export CYLC_VERSION=8 ; bash -l cylc vip -z g=groups -n test_name -S USE_MIRRORS=true -S USE_HEADS=true /path/to/clone/rose-stem ", + ), ] @pytest.mark.parametrize( - ("version", "expected"), - [test_data for test_data in data_major_cylc_version], + ("suite", "clone_path", "version", "name", "expected"), + [test_data for test_data in data_generate_cylc_command], ) -def test_major_cylc_version(version, expected): - assert major_cylc_version(version) == expected +def test_generate_cylc_command(suite, clone_path, version, name, expected): + assert generate_cylc_command(suite, clone_path, version, name) == expected