[arteval-bench] Add Neutrino (OSDI'25) artifact

benchmarks/arteval_bench/data/benchmark/osdi25_neutrino/_agent_eval/main.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+"""Runs environment setup, build, benchmark prep, and experiment runs checks for Neutrino (OSDI'25)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Dict, Tuple
+import os
+import sys
+
+
+from evaluator.utils import (  # noqa: E402
+    EntryConfig,
+    LoggerConfig,
+    get_logger,
+    record_result,
+)
+
+from oracle_artifact_build import OracleArtifactBuild  # noqa: E402
+from oracle_benchmark_prep import OracleBenchmarkPrep  # noqa: E402
+from oracle_env_setup import OracleEnvSetup  # noqa: E402
+from oracle_experiment_runs import OracleExperimentRuns  # noqa: E402
+
+
+def _resolve_workspace_paths() -> Tuple[Path, Path, Path]:
+    """Resolve and validate _agent_eval/ and neutrino/ locations.
+
+    Expects either:
+      (1) _agent_eval/ and the Neutrino repo are located in the same workspace root; or
+      (2) _AGENT_EVAL_DIR and _NEUTRINO_HOME are set by the user.
+    """
+    try:
+        env_agent_eval = os.environ.get("_AGENT_EVAL_DIR")
+        env_neutrino_home = os.environ.get("_NEUTRINO_HOME")
+
+        agent_eval_dir = (
+            Path(env_agent_eval).expanduser().resolve()
+            if env_agent_eval
+            else Path(__file__).resolve().parent
+        )
+
+        workspace_root = (
+            Path(env_neutrino_home).expanduser().resolve()
+            if env_neutrino_home
+            else agent_eval_dir.parent.resolve()
+        )
+
+        if not agent_eval_dir.is_dir():
+            raise RuntimeError(
+                f"Invalid _agent_eval dir: {agent_eval_dir}\n"
+                "Set _AGENT_EVAL_DIR to the directory containing main.py if needed."
+            )
+
+        neutrino_repo_root = workspace_root / "neutrino"
+        if not neutrino_repo_root.is_dir():
+            raise RuntimeError(
+                f"Invalid Neutrino workspace: {workspace_root}\n"
+                f"Expected to find a Neutrino repository directory at: {neutrino_repo_root}\n"
+                "This runner expects _agent_eval/ and the Neutrino repo to be located in the same workspace root.\n"
+                "Set _NEUTRINO_HOME to the workspace root if needed."
+            )
+
+        return agent_eval_dir, workspace_root, neutrino_repo_root
+
+    except OSError as exc:
+        raise RuntimeError(f"Failed to resolve workspace paths: {exc}") from exc
+
+
+def _build_neutrino_config(
+    *, agent_eval_dir: Path, workspace_root: Path, neutrino_repo_root: Path
+) -> EntryConfig:
+    """Constructs EntryConfig for the Neutrino evaluation bundle from resolved paths."""
+
+    return EntryConfig(
+        name="osdi25-neutrino",
+        home_dir=workspace_root,
+        repository_paths={
+            "osdi25-neutrino": neutrino_repo_root,
+        },
+        results_paths={
+            # Need to add results dir
+        },
+        ground_truth_paths={
+            # Need _agent_eval/refs.
+        },
+        similarity_ratio=0.75,
+    )
+
+
+def main(argv: list[str]) -> int:
+    verbose = "--verbose" in argv
+
+    results: Dict[str, int] = {}
+    score = 0
+
+    logger_name = os.environ.get("EVAL_LOGGER_NAME", "NEUTRINO-AGENT-EVALUATOR")
+    logger = get_logger(LoggerConfig(root_name=logger_name))
+
+    try:
+        agent_eval_dir, workspace_root, neutrino_repo_root = _resolve_workspace_paths()
+        NEUTRINO_CONFIG = _build_neutrino_config(
+            agent_eval_dir=agent_eval_dir,
+            workspace_root=workspace_root,
+            neutrino_repo_root=neutrino_repo_root,
+        )
+    except RuntimeError as exc:
+        raise SystemExit(str(exc)) from exc
+
+    env_checker = OracleEnvSetup(config=NEUTRINO_CONFIG, logger=logger)
+    score += record_result(results, type(env_checker).__name__, env_checker.run(verbose=verbose))
+
+    build_checker = OracleArtifactBuild(config=NEUTRINO_CONFIG, logger=logger)
+    score += record_result(results, type(build_checker).__name__, build_checker.run(verbose=verbose))
+
+    prep_checker = OracleBenchmarkPrep(config=NEUTRINO_CONFIG, logger=logger)
+    score += record_result(results, type(prep_checker).__name__, prep_checker.run(verbose=verbose))
+
+    runs_checker = OracleExperimentRuns(config=NEUTRINO_CONFIG, logger=logger)
+    score += record_result(results, type(runs_checker).__name__, runs_checker.run(verbose=verbose))
+
+    logger.info("Agent scores: %s", results)
+    return score
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv[1:]))

benchmarks/arteval_bench/data/benchmark/osdi25_neutrino/_agent_eval/oracle_artifact_build.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+"""Artifact build oracle for Neutrino (OSDI'25).
+
+Validates:
+  - Repository working directory exists.
+  - The Neutrino CLI is on PATH and can invoke `--help`.
+  - The Neutrino module is importable after installation.
+"""
+
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+import logging
+from pathlib import Path
+import sys
+
+from evaluator.oracle_artifact_build_primitives import (
+    BuildCommandRequirement,
+    OracleArtifactBuildBase,
+)
+from evaluator.utils import BaseRequirement, EntryConfig
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class BuildTarget:
+    """Declarative description of one build command to run."""
+
+    name: str
+    cmd: Sequence[str]
+    relative_workdir: Path | None = None
+    optional: bool = False
+    timeout_seconds: float = 60.0
+    env_overrides: Mapping[str, str] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if not self.name:
+            raise ValueError("BuildTarget.name must be non-empty")
+
+        if isinstance(self.cmd, (str, bytes)) or not self.cmd:
+            raise ValueError("BuildTarget.cmd must be a non-empty argv sequence")
+
+        object.__setattr__(self, "cmd", tuple(self.cmd))
+
+        if self.relative_workdir is not None and not isinstance(self.relative_workdir, Path):
+            object.__setattr__(self, "relative_workdir", Path(self.relative_workdir))
+
+
+class OracleArtifactBuild(OracleArtifactBuildBase):
+    """The artifact build oracle for Neutrino."""
+
+    def __init__(
+        self,
+        *,
+        config: EntryConfig,
+        logger: logging.Logger,
+        targets: Sequence[BuildTarget] | None = None,
+    ) -> None:
+        super().__init__(logger=logger)
+        self._config = config
+
+        if targets is None:
+            targets = self._make_default_targets()
+        self._targets = tuple(targets)
+
+        names = [t.name for t in self._targets]
+        if len(names) != len(set(names)):
+            raise ValueError(f"Duplicate build target names: {names!r}")
+
+    def _make_default_targets(self) -> tuple[BuildTarget, ...]:
+        py = sys.executable or "python"
+
+        return (
+            BuildTarget(
+                name="neutrino: import test",
+                cmd=(py, "-c", "import neutrino; print(neutrino.__file__)"),
+                timeout_seconds=30.0,
+            ),
+            BuildTarget(
+                name="neutrino: CLI help (optional)",
+                cmd=("neutrino", "--help"),
+                optional=True,
+                timeout_seconds=30.0,
+            ),
+        )
+
+    def requirements(self) -> Sequence[BaseRequirement]:
+        """Returns an ordered list of build requirements to validate."""
+        repo_root = self._config.repository_paths.get(self._config.name)
+
+        if repo_root is None:
+            return (
+                BuildCommandRequirement(
+                    name=f"config: missing repository_paths entry for {self._config.name!r}",
+                    optional=False,
+                    cwd=Path(self._config.home_dir) / "__MISSING_REPOSITORY_PATH__",
+                    cmd=("true",),
+                    timeout_seconds=30.0,
+                ),
+            )
+
+        return tuple(
+            BuildCommandRequirement(
+                name=target.name,
+                optional=target.optional,
+                cwd=repo_root,
+                cmd=target.cmd,
+                relative_workdir=target.relative_workdir,
+                timeout_seconds=target.timeout_seconds,
+                env_overrides=target.env_overrides,
+            )
+            for target in self._targets
+        )

benchmarks/arteval_bench/data/benchmark/osdi25_neutrino/_agent_eval/oracle_env_setup.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+"""Environment setup oracle for Neutrino (OSDI'25).
+
+Validates:
+  - Baseline tools for running the (static) evaluation workflow.
+  - Repository directory layout and required artifact files.
+  - Static and dynamic evaluation prerequisites.
+"""
+
+import logging
+from pathlib import Path
+from typing import Mapping, Sequence
+
+from evaluator import utils
+from evaluator.utils import EntryConfig
+from evaluator.oracle_env_setup_primitives import (
+    DependencyVersionRequirement,
+    FilesystemPathRequirement,
+    OracleEnvSetupBase,
+    PathType,
+    VersionCompare,
+)
+
+
+def _required_path(paths: Mapping[str, Path], key: str, *, label: str) -> Path:
+    """Returns a required path from a mapping with a clear error."""
+    try:
+        return paths[key]
+    except KeyError as e:
+        raise ValueError(f"Missing {label}[{key!r}] in EntryConfig") from e
+
+
+class OracleEnvSetup(OracleEnvSetupBase):
+    """Validates environment prerequisites for Neutrino (OSDI'25)."""
+
+    def __init__(self, *, config: EntryConfig, logger: logging.Logger) -> None:
+        super().__init__(logger)
+        self._config = config
+
+    def requirements(self) -> Sequence[utils.BaseRequirement]:
+        repo_root = _required_path(
+            self._config.repository_paths, self._config.name, label="repository_paths"
+        )
+
+        artifact_dir = repo_root / "artifact"
+        pkg_dir = repo_root / "neutrino"
+
+        # Static evaluation requirements
+        reqs: list[utils.BaseRequirement] = [
+            DependencyVersionRequirement(
+                name="python",
+                cmd=("python", "--version"),
+                required_version=(3, 11, 0),
+                compare=VersionCompare.GEQ,
+            ),
+            DependencyVersionRequirement(
+                name="pip",
+                cmd=("python", "-m", "pip", "--version"),
+                required_version=(0, 0, 0),
+                compare=VersionCompare.GEQ,
+            ),
+            DependencyVersionRequirement(
+                name="wget",
+                cmd=("wget", "--version"),
+                required_version=(0, 0, 0),
+                compare=VersionCompare.GEQ,
+                optional=True,
+            ),
+            DependencyVersionRequirement(
+                name="unzip",
+                cmd=("unzip", "-v"),
+                required_version=(0, 0, 0),
+                compare=VersionCompare.GEQ,
+                optional=True,
+            ),
+            FilesystemPathRequirement(
+                name="repo_root_exists",
+                path=repo_root,
+                path_type=PathType.DIRECTORY,
+            ),
+            FilesystemPathRequirement(
+                name="artifact_dir_exists",
+                path=artifact_dir,
+                path_type=PathType.DIRECTORY,
+            ),
+            FilesystemPathRequirement(
+                name="static_notebook_exists",
+                path=artifact_dir / "static.ipynb",
+                path_type=PathType.FILE,
+            ),
+        ]
+
+        # Dynamic evaluation requirements
+        reqs.extend(
+            [
+                DependencyVersionRequirement(
+                    name="gcc",
+                    cmd=("gcc", "--version"),
+                    required_version=(0, 0, 0),
+                    compare=VersionCompare.GEQ,
+                    optional=False,
+                ),
+                DependencyVersionRequirement(
+                    name="nm",
+                    cmd=("nm", "--version"),
+                    required_version=(0, 0, 0),
+                    compare=VersionCompare.GEQ,
+                    optional=False,
+                ),
+                DependencyVersionRequirement(
+                    name="cmake",
+                    cmd=("cmake", "--version"),
+                    required_version=(0, 0, 0),
+                    compare=VersionCompare.GEQ,
+                    optional=False,
+                ),
+                DependencyVersionRequirement(
+                    name="make",
+                    cmd=("make", "--version"),
+                    required_version=(0, 0, 0),
+                    compare=VersionCompare.GEQ,
+                    optional=False,
+                ),
+                DependencyVersionRequirement(
+                    name="nvidia-smi",
+                    cmd=("nvidia-smi",),
+                    required_version=(0, 0, 0),
+                    compare=VersionCompare.GEQ,
+                    optional=True,
+                ),
+                DependencyVersionRequirement(
+                    name="ptxas",
+                    cmd=("ptxas", "--version"),
+                    required_version=(0, 0, 0),
+                    compare=VersionCompare.GEQ,
+                    optional=True,
+                ),
+                DependencyVersionRequirement(
+                    name="cuobjdump",
+                    cmd=("cuobjdump", "--version"),
+                    required_version=(0, 0, 0),
+                    compare=VersionCompare.GEQ,
+                    optional=True,
+                ),
+                FilesystemPathRequirement(
+                    name="dynamic_notebook_exists",
+                    path=artifact_dir / "dynamic.ipynb",
+                    path_type=PathType.FILE,
+                ),
+            ]
+        )
+
+        return reqs

benchmarks/arteval_bench/data/benchmark/osdi25_neutrino/neutrino/MANIFEST.in

@@ -0,0 +1 @@
+recursive-include neutrino *