core/tests/scripts/check_requirements/test_runner.py

"""Tests for script.check_requirements.runner."""

import json

import pytest

from script.check_requirements.models import CheckKind, CheckStatus
from script.check_requirements.pypi import (
    ProvenanceResult,
    PypiPackageInfo,
    Vulnerability,
)
from script.check_requirements.runner import run_checks


def _patch_pypi(
    monkeypatch: pytest.MonkeyPatch,
    pypi_info: PypiPackageInfo,
    prov: ProvenanceResult,
) -> None:
    monkeypatch.setattr(
        "script.check_requirements.runner.fetch_package_info",
        lambda name, version: pypi_info,
    )
    monkeypatch.setattr(
        "script.check_requirements.runner.check_provenance", lambda info: prov
    )


def test_runner_attestation_recognised(monkeypatch: pytest.MonkeyPatch) -> None:
    """Recognised attestation → ci_upload PASS, release_pipeline PASS, repo + pr_link needs_agent."""
    _patch_pypi(
        monkeypatch,
        PypiPackageInfo(
            project_urls={"Source": "https://github.com/example/pkg"},
            repo_url="https://github.com/example/pkg",
            file_provenance_urls=["whatever"],
            found=True,
        ),
        ProvenanceResult(
            has_attestation=True,
            publisher_kind="GitHub",
            recognized_publisher=True,
            detail="Trusted Publisher attestation found (GitHub).",
        ),
    )
    diff = (
        "diff --git a/requirements_all.txt b/requirements_all.txt\n"
        "--- a/requirements_all.txt\n"
        "+++ b/requirements_all.txt\n"
        "@@ -1 +1 @@\n"
        "-pkg==1.0.0\n"
        "+pkg==1.1.0\n"
    )
    result = run_checks(pr_number=42, diff_text=diff)
    assert len(result.packages) == 1
    pkg = result.packages[0]
    assert pkg.checks[CheckKind.CI_UPLOAD].status == CheckStatus.PASS
    assert pkg.checks[CheckKind.RELEASE_PIPELINE].status == CheckStatus.PASS
    assert pkg.checks[CheckKind.REPO_PUBLIC].status == CheckStatus.NEEDS_AGENT
    assert pkg.checks[CheckKind.PR_LINK].status == CheckStatus.NEEDS_AGENT
    assert pkg.checks[CheckKind.ASYNC_BLOCKING].status == CheckStatus.NEEDS_AGENT
    assert result.needs_agent is True


def test_runner_no_attestation(monkeypatch: pytest.MonkeyPatch) -> None:
    """No attestation → ci_upload WARN, release_pipeline NEEDS_AGENT."""
    _patch_pypi(
        monkeypatch,
        PypiPackageInfo(
            project_urls={"Source": "https://github.com/example/pkg"},
            repo_url="https://github.com/example/pkg",
            file_provenance_urls=[],
            found=True,
        ),
        ProvenanceResult(
            has_attestation=False,
            publisher_kind=None,
            recognized_publisher=False,
            detail="No PEP 740 provenance attestation present on PyPI.",
        ),
    )
    diff = (
        "diff --git a/requirements_all.txt b/requirements_all.txt\n"
        "--- a/requirements_all.txt\n"
        "+++ b/requirements_all.txt\n"
        "@@ -1 +1 @@\n"
        "-pkg==1.0.0\n"
        "+pkg==1.1.0\n"
    )
    result = run_checks(pr_number=1, diff_text=diff)
    pkg = result.packages[0]
    assert pkg.checks[CheckKind.CI_UPLOAD].status == CheckStatus.WARN
    assert pkg.checks[CheckKind.RELEASE_PIPELINE].status == CheckStatus.NEEDS_AGENT


def test_runner_attestation_present_but_publisher_unrecognised(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Attestation present but publisher unknown → ci_upload WARN, release_pipeline NEEDS_AGENT."""
    _patch_pypi(
        monkeypatch,
        PypiPackageInfo(
            project_urls={"Source": "https://github.com/example/pkg"},
            repo_url="https://github.com/example/pkg",
            file_provenance_urls=["whatever"],
            found=True,
        ),
        ProvenanceResult(
            has_attestation=True,
            publisher_kind="AcmeCI",
            recognized_publisher=False,
            detail="Attestation present but publisher kind 'AcmeCI' is not recognised.",
        ),
    )
    diff = (
        "diff --git a/requirements_all.txt b/requirements_all.txt\n"
        "--- a/requirements_all.txt\n"
        "+++ b/requirements_all.txt\n"
        "@@ -1 +1 @@\n"
        "-pkg==1.0.0\n"
        "+pkg==1.1.0\n"
    )
    result = run_checks(pr_number=1, diff_text=diff)
    pkg = result.packages[0]
    assert pkg.checks[CheckKind.CI_UPLOAD].status == CheckStatus.WARN
    assert pkg.checks[CheckKind.RELEASE_PIPELINE].status == CheckStatus.NEEDS_AGENT
    assert "publisher unrecognised" in pkg.checks[CheckKind.RELEASE_PIPELINE].details


def test_runner_marks_missing_version_as_fail(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A version that doesn't exist on PyPI must FAIL, not request the agent."""
    _patch_pypi(
        monkeypatch,
        PypiPackageInfo(
            project_urls={},
            repo_url=None,
            file_provenance_urls=[],
            found=False,
        ),
        ProvenanceResult(
            has_attestation=False,
            publisher_kind=None,
            recognized_publisher=False,
            detail="Version not found on PyPI.",
        ),
    )
    diff = (
        "diff --git a/requirements_all.txt b/requirements_all.txt\n"
        "--- a/requirements_all.txt\n"
        "+++ b/requirements_all.txt\n"
        "@@ -1 +1 @@\n"
        "-pkg==1.0.0\n"
        "+pkg==9.9.9\n"
    )
    result = run_checks(pr_number=1, diff_text=diff)
    pkg = result.packages[0]
    assert pkg.checks[CheckKind.CI_UPLOAD].status == CheckStatus.FAIL
    assert pkg.checks[CheckKind.RELEASE_PIPELINE].status == CheckStatus.FAIL
    # No repo URL → repo_public, pr_link and async_blocking short-circuit to FAIL
    assert pkg.checks[CheckKind.REPO_PUBLIC].status == CheckStatus.FAIL
    assert pkg.checks[CheckKind.PR_LINK].status == CheckStatus.FAIL
    assert pkg.checks[CheckKind.ASYNC_BLOCKING].status == CheckStatus.FAIL
    assert result.needs_agent is False


def test_runner_pypi_found_but_no_repo_url_fails_repo_checks(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A package on PyPI without a source URL fails repo_public and pr_link."""
    _patch_pypi(
        monkeypatch,
        PypiPackageInfo(
            project_urls={},
            repo_url=None,
            file_provenance_urls=[],
            found=True,
        ),
        ProvenanceResult(
            has_attestation=False,
            publisher_kind=None,
            recognized_publisher=False,
            detail="No PEP 740 provenance attestation present on PyPI.",
        ),
    )
    diff = (
        "diff --git a/requirements_all.txt b/requirements_all.txt\n"
        "--- a/requirements_all.txt\n"
        "+++ b/requirements_all.txt\n"
        "@@ -1 +1 @@\n"
        "-pkg==1.0.0\n"
        "+pkg==1.1.0\n"
    )
    result = run_checks(pr_number=1, diff_text=diff)
    pkg = result.packages[0]
    assert pkg.checks[CheckKind.REPO_PUBLIC].status == CheckStatus.FAIL
    assert pkg.checks[CheckKind.PR_LINK].status == CheckStatus.FAIL
    assert pkg.checks[CheckKind.ASYNC_BLOCKING].status == CheckStatus.FAIL
    assert "does not advertise" in pkg.checks[CheckKind.REPO_PUBLIC].details


def test_runner_async_blocking_new_package_full_review(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A newly added package asks the agent for a full-tree async review."""
    _patch_pypi(
        monkeypatch,
        PypiPackageInfo(
            project_urls={"Source": "https://github.com/x/y"},
            repo_url="https://github.com/x/y",
            file_provenance_urls=["whatever"],
            found=True,
        ),
        ProvenanceResult(
            has_attestation=True,
            publisher_kind="GitHub",
            recognized_publisher=True,
            detail="ok",
        ),
    )
    diff = (
        "diff --git a/requirements_all.txt b/requirements_all.txt\n"
        "--- a/requirements_all.txt\n"
        "+++ b/requirements_all.txt\n"
        "@@ -0,0 +1 @@\n"
        "+pkg==1.0.0\n"
    )
    result = run_checks(pr_number=1, diff_text=diff)
    pkg = result.packages[0]
    assert pkg.old_version is None
    detail = pkg.checks[CheckKind.ASYNC_BLOCKING].details
    assert pkg.checks[CheckKind.ASYNC_BLOCKING].status == CheckStatus.NEEDS_AGENT
    assert "New dependency" in detail
    assert "entire source tree" in detail


def test_runner_async_blocking_version_bump_diff_only(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A version bump asks the agent to review only the diff for new blocking calls."""
    _patch_pypi(
        monkeypatch,
        PypiPackageInfo(
            project_urls={"Source": "https://github.com/x/y"},
            repo_url="https://github.com/x/y",
            file_provenance_urls=["whatever"],
            found=True,
        ),
        ProvenanceResult(
            has_attestation=True,
            publisher_kind="GitHub",
            recognized_publisher=True,
            detail="ok",
        ),
    )
    diff = (
        "diff --git a/requirements_all.txt b/requirements_all.txt\n"
        "--- a/requirements_all.txt\n"
        "+++ b/requirements_all.txt\n"
        "@@ -1 +1 @@\n"
        "-pkg==1.0.0\n"
        "+pkg==1.1.0\n"
    )
    result = run_checks(pr_number=1, diff_text=diff)
    pkg = result.packages[0]
    assert pkg.old_version == "1.0.0"
    detail = pkg.checks[CheckKind.ASYNC_BLOCKING].details
    assert pkg.checks[CheckKind.ASYNC_BLOCKING].status == CheckStatus.NEEDS_AGENT
    assert "1.0.0" in detail and "1.1.0" in detail
    assert "diff" in detail


def test_runner_yanked_release_fails(monkeypatch: pytest.MonkeyPatch) -> None:
    """A yanked release on PyPI must FAIL the yanked check."""
    _patch_pypi(
        monkeypatch,
        PypiPackageInfo(
            project_urls={"Source": "https://github.com/x/y"},
            repo_url="https://github.com/x/y",
            file_provenance_urls=["whatever"],
            found=True,
            yanked=True,
            yanked_reason="critical bug",
        ),
        ProvenanceResult(
            has_attestation=True,
            publisher_kind="GitHub",
            recognized_publisher=True,
            detail="ok",
        ),
    )
    diff = (
        "diff --git a/requirements_all.txt b/requirements_all.txt\n"
        "--- a/requirements_all.txt\n"
        "+++ b/requirements_all.txt\n"
        "@@ -1 +1 @@\n"
        "-pkg==1.0.0\n"
        "+pkg==1.1.0\n"
    )
    result = run_checks(pr_number=1, diff_text=diff)
    pkg = result.packages[0]
    assert pkg.checks[CheckKind.YANKED].status == CheckStatus.FAIL
    assert "yanked" in pkg.checks[CheckKind.YANKED].details
    assert "critical bug" in pkg.checks[CheckKind.YANKED].details


def test_runner_non_yanked_release_passes(monkeypatch: pytest.MonkeyPatch) -> None:
    """A normal (non-yanked) release passes the yanked check."""
    _patch_pypi(
        monkeypatch,
        PypiPackageInfo(
            project_urls={"Source": "https://github.com/x/y"},
            repo_url="https://github.com/x/y",
            file_provenance_urls=["whatever"],
            found=True,
        ),
        ProvenanceResult(
            has_attestation=True,
            publisher_kind="GitHub",
            recognized_publisher=True,
            detail="ok",
        ),
    )
    diff = (
        "diff --git a/requirements_all.txt b/requirements_all.txt\n"
        "--- a/requirements_all.txt\n"
        "+++ b/requirements_all.txt\n"
        "@@ -1 +1 @@\n"
        "-pkg==1.0.0\n"
        "+pkg==1.1.0\n"
    )
    result = run_checks(pr_number=1, diff_text=diff)
    pkg = result.packages[0]
    assert pkg.checks[CheckKind.YANKED].status == CheckStatus.PASS


def test_runner_active_vulnerabilities_fail(monkeypatch: pytest.MonkeyPatch) -> None:
    """An active CVE/GHSA on PyPI fails the vulnerabilities check with a link."""
    _patch_pypi(
        monkeypatch,
        PypiPackageInfo(
            project_urls={"Source": "https://github.com/x/y"},
            repo_url="https://github.com/x/y",
            file_provenance_urls=["whatever"],
            found=True,
            vulnerabilities=[
                Vulnerability(
                    id="GHSA-xxxx-xxxx-xxxx",
                    aliases=("CVE-2099-12345",),
                    summary="rce",
                    fixed_in=("1.2.0",),
                    link="https://osv.dev/vulnerability/GHSA-xxxx-xxxx-xxxx",
                ),
            ],
        ),
        ProvenanceResult(
            has_attestation=True,
            publisher_kind="GitHub",
            recognized_publisher=True,
            detail="ok",
        ),
    )
    diff = (
        "diff --git a/requirements_all.txt b/requirements_all.txt\n"
        "--- a/requirements_all.txt\n"
        "+++ b/requirements_all.txt\n"
        "@@ -1 +1 @@\n"
        "-pkg==1.0.0\n"
        "+pkg==1.1.0\n"
    )
    result = run_checks(pr_number=1, diff_text=diff)
    pkg = result.packages[0]
    assert pkg.checks[CheckKind.VULNERABILITIES].status == CheckStatus.FAIL
    details = pkg.checks[CheckKind.VULNERABILITIES].details
    # CVE alias is preferred as the link label when present.
    assert "[CVE-2099-12345](" in details
    assert "[GHSA-xxxx-xxxx-xxxx](" not in details
    assert "fixed in: 1.2.0" in details
    assert "https://osv.dev/vulnerability/GHSA-xxxx-xxxx-xxxx" in details


def test_runner_no_vulnerabilities_passes(monkeypatch: pytest.MonkeyPatch) -> None:
    """An empty `vulnerabilities` list passes the check."""
    _patch_pypi(
        monkeypatch,
        PypiPackageInfo(
            project_urls={"Source": "https://github.com/x/y"},
            repo_url="https://github.com/x/y",
            file_provenance_urls=["whatever"],
            found=True,
        ),
        ProvenanceResult(
            has_attestation=True,
            publisher_kind="GitHub",
            recognized_publisher=True,
            detail="ok",
        ),
    )
    diff = (
        "diff --git a/requirements_all.txt b/requirements_all.txt\n"
        "--- a/requirements_all.txt\n"
        "+++ b/requirements_all.txt\n"
        "@@ -1 +1 @@\n"
        "-pkg==1.0.0\n"
        "+pkg==1.1.0\n"
    )
    result = run_checks(pr_number=1, diff_text=diff)
    pkg = result.packages[0]
    assert pkg.checks[CheckKind.VULNERABILITIES].status == CheckStatus.PASS


def test_runner_serialises_to_json(monkeypatch: pytest.MonkeyPatch) -> None:
    """The artifact contract: `to_dict()` is JSON-serialisable with expected keys."""
    _patch_pypi(
        monkeypatch,
        PypiPackageInfo(
            project_urls={"Source": "https://github.com/x/y"},
            repo_url="https://github.com/x/y",
            file_provenance_urls=["whatever"],
            found=True,
        ),
        ProvenanceResult(
            has_attestation=True,
            publisher_kind="GitHub",
            recognized_publisher=True,
            detail="ok",
        ),
    )
    diff = (
        "diff --git a/requirements_all.txt b/requirements_all.txt\n"
        "--- a/requirements_all.txt\n"
        "+++ b/requirements_all.txt\n"
        "@@ -1 +1 @@\n"
        "-pkg==1.0.0\n"
        "+pkg==1.1.0\n"
    )
    result = run_checks(pr_number=42, diff_text=diff)
    serialised = json.dumps(result.to_dict())
    assert '"rendered_comment"' in serialised
    assert '"needs_agent"' in serialised
    assert '"checks"' in serialised
    assert '"repo_public"' in serialised  # check kinds are in the JSON