feat(eval): add --keep-conversations flag

packages/gooddata-eval/src/gooddata_eval/cli/main.py

@@ -15,7 +15,12 @@
 from rich.table import Table
 
 from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, UNGATED_AGENTIC_TEST_KINDS, run_agentic_items
-from gooddata_eval.core.chat.sse_client import ChatClient, set_default_item_timeout, set_default_turn_timeout
+from gooddata_eval.core.chat.sse_client import (
+    ChatClient,
+    keep_conversations,
+    set_default_item_timeout,
+    set_default_turn_timeout,
+)
 from gooddata_eval.core.config import (
     DEFAULT_GATE,
     DEFAULT_JUDGE_MODEL,
@@ -169,7 +174,17 @@ def _build_parser() -> argparse.ArgumentParser:
         "--preserve-failed",
         action="store_true",
         dest="preserve_failed",
-        help="Keep failed conversations on the server for post-mortem inspection.",
+        help="Keep failed conversations on the server for post-mortem inspection. Single-turn "
+        "kinds only; use --keep-conversations to cover the agentic ones.",
+    )
+    run.add_argument(
+        "--keep-conversations",
+        action="store_true",
+        dest="keep_conversations",
+        help="Keep every conversation on the server, passed or failed, so the AI Interaction "
+        "Intelligence endpoints can be queried after the run (or set "
+        "GOODDATA_EVAL_KEEP_CONVERSATIONS=1). Covers the agentic kinds, which --preserve-failed "
+        "does not. Leaves state behind: only for a diagnostic run.",
     )
     run.add_argument(
         "--reasoning-effort",
@@ -723,13 +738,17 @@ def main(argv: list[str] | None = None) -> int:
             quiet=args.quiet,
             kind=args.kind,
             preserve_failed=args.preserve_failed,
+            keep_conversations=args.keep_conversations,
             reasoning_effort=args.reasoning_effort,
             gate=normalize_gate(args.gate),
             agent_id=args.agent_id or os.environ.get("GD_EVAL_AGENT_ID"),
             turn_timeout_s=args.turn_timeout,
             item_timeout_s=args.item_timeout,
         )
-        return _run(config)
+        # Scoped to this run: main() is re-entrant under test and as a library call, and a
+        # leaked True would leave the next run's conversations on the server unasked.
+        with keep_conversations(config.keep_conversations):
+            return _run(config)
     except (
         ConnectionError_,
         ModelResolutionError,

packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py

@@ -13,14 +13,15 @@
 protocol, not on this class.
 """
 
+import contextlib
 import functools
 import json
 import logging
 import os
 import re
 import time
 from dataclasses import dataclass, field
-from typing import Any, Callable, Iterable, TypeVar
+from typing import Any, Callable, Iterable, Iterator, TypeVar
 from urllib.parse import quote
 
 import httpx
@@ -119,6 +120,16 @@ def _float_env(name: str, default: float) -> float:
     return float(raw) if raw else default
 
 
+def _bool_env(name: str, default: bool) -> bool:
+    """Read a flag from the environment, falling back to ``default`` when unset or blank.
+
+    ``0``/``false``/``no``/``off`` are false, anything else present is true -- so the usual
+    ``VAR=0`` turns a flag off rather than reading as a non-empty, therefore true, string.
+    """
+    raw = os.getenv(name)
+    return raw.strip().lower() not in ("0", "false", "no", "off") if raw and raw.strip() else default
+
+
 # Retry budget defaults, giving a ~2 min worst-case cap per send (5/10/20/40/60s).
 # Each is overridable via env so CI can retune without cutting a new release --
 # read per call rather than at import, so an exported value cannot silently
@@ -148,6 +159,13 @@ def _float_env(name: str, default: float) -> float:
 _TRACE_LABELS_ENV = "GOODDATA_EVAL_TRACE_LABELS"
 _TRACE_LABEL_KEY = re.compile(r"[A-Za-z0-9_-]+")
 
+# When set, no conversation is deleted -- pass or fail, single-turn or agentic. The
+# server-side record is what the Interaction Intelligence endpoints read, so a run meant to
+# be inspected afterwards has to leave it behind. Enforced inside delete_conversation rather
+# than at each call site: thirteen agentic evaluators call it twenty times between them, and
+# a fourteenth kind would have been one more place to remember.
+_KEEP_CONVERSATIONS = _bool_env("GOODDATA_EVAL_KEEP_CONVERSATIONS", False)
+
 
 @functools.cache
 def _warn_skipped_trace_label(key: str) -> None:
@@ -194,6 +212,37 @@ def set_default_item_timeout(seconds: float | None) -> None:
     _ITEM_TIMEOUT_S = seconds or 0.0
 
 
+def set_keep_conversations(keep: bool) -> None:
+    """Keep every conversation this process creates, whatever its outcome.
+
+    Lands here for the same reason the timeout defaults do -- the agentic evaluators build
+    their own clients deep in the call tree -- but takes effect at deletion rather than at
+    construction, so a client already built honours it too.
+
+    Prefer ``keep_conversations`` for a single run: a bare set leaks into whatever the
+    process does next, and the direction of the leak is the costly one -- a later run that
+    asked for nothing would silently leave server-side state behind.
+    """
+    global _KEEP_CONVERSATIONS
+    _KEEP_CONVERSATIONS = keep
+
+
+@contextlib.contextmanager
+def keep_conversations(keep: bool) -> Iterator[None]:
+    """Scope conversation retention to one run, restoring the previous setting on exit.
+
+    ``keep=False`` is not "delete": it leaves the setting alone, so an operator who exported
+    GOODDATA_EVAL_KEEP_CONVERSATIONS still gets it on a run that passes no flag.
+    """
+    previous = _KEEP_CONVERSATIONS
+    if keep:
+        set_keep_conversations(True)
+    try:
+        yield
+    finally:
+        set_keep_conversations(previous)
+
+
 T = TypeVar("T")
 
 
@@ -550,6 +599,8 @@ def _do() -> str:
         return conversation_id
 
     def delete_conversation(self, conversation_id: str) -> None:
+        if _KEEP_CONVERSATIONS:
+            return
         try:
             self._client.delete(f"{self._base}/{conversation_id}", headers=self._auth)
         except httpx.HTTPError:

packages/gooddata-eval/src/gooddata_eval/core/config.py

@@ -90,6 +90,9 @@ class RunConfig:
     quiet: bool = False
     kind: str = "visualization"
     preserve_failed: bool = False
+    keep_conversations: bool = False
+    """Keep every conversation, pass or fail. Superset of ``preserve_failed``; applies to the
+    agentic kinds too, which ``preserve_failed`` never reached."""
     reasoning_effort: ReasoningEffort | None = None
     agent_id: str | None = None
     turn_timeout_s: float | None = None

packages/gooddata-eval/tests/test_agentic_runner.py

@@ -1,9 +1,11 @@
 # (C) 2026 GoodData Corporation. All rights reserved.
 # SPDX-License-Identifier: LicenseRef-GoodData-Enterprise
 import importlib
+import re
 import threading
 import time
 from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
 from typing import Any
 from unittest.mock import patch
 
@@ -855,3 +857,37 @@ def test_run_agentic_binds_the_user_context_to_its_chat_client(
     with patch.object(module, "ChatClient", side_effect=_Stop) as mock_client, pytest.raises(_Stop):
         getattr(module, run_fn)("https://h", "tok", "ws1", "q", expected, user_context=user_context)
     assert mock_client.call_args.kwargs["user_context"] == user_context
+
+
+def _agentic_module_sources() -> dict[str, str]:
+    """Every agentic evaluator module's source, keyed by name. Discovered by scanning the
+    package so a new kind is covered the day it lands rather than when someone remembers."""
+    package = importlib.import_module("gooddata_eval.core.agentic")
+    directory = Path(package.__file__).parent
+    return {
+        path.stem: path.read_text(encoding="utf-8")
+        for path in sorted(directory.glob("*.py"))
+        if not path.stem.startswith("_")
+    }
+
+
+@pytest.mark.parametrize("module_name", sorted(_agentic_module_sources()))
+def test_every_agentic_module_deletes_through_the_client_method(module_name: str) -> None:
+    """--keep-conversations is enforced inside ChatClient.delete_conversation, so a module
+    that issues its own DELETE would quietly ignore it. Thirteen modules delete twenty times
+    between them; one bypass is enough to lose the conversation a diagnostic run needed."""
+    source = _agentic_module_sources()[module_name]
+    # Only HTTP deletes: `created.delete(sdk, workspace_id)` is workspace-object cleanup,
+    # which has nothing to do with conversation retention.
+    bypasses = [
+        line.strip() for line in source.splitlines() if re.search(r"\b(httpx|requests|_client)\.delete\(", line)
+    ]
+    assert not bypasses, f"{module_name} issues its own DELETE; call client.delete_conversation instead: {bypasses}"
+
+
+def test_the_agentic_modules_do_delete_conversations() -> None:
+    """Guards the guard: an assertion that only forbids a pattern still passes once the
+    cleanup it is protecting has been deleted outright."""
+    sources = _agentic_module_sources()
+    deleting = {name for name, src in sources.items() if "delete_conversation(" in src}
+    assert len(deleting) >= 12, f"expected most agentic kinds to clean up, found {sorted(deleting)}"

packages/gooddata-eval/tests/test_cli.py

@@ -1,4 +1,5 @@
 # (C) 2026 GoodData Corporation
+import contextlib
 import io
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
@@ -683,6 +684,74 @@ def _fake_run(items, backend, *, runs, model, workspace_id, **kw):
     assert captured_kwargs.get("preserve_failed") is True
 
 
+def _recording_keep(applied: list[bool]):
+    """Stand-in for the keep_conversations context manager that records what it was asked for."""
+
+    @contextlib.contextmanager
+    def _cm(keep: bool):
+        applied.append(keep)
+        yield
+
+    return _cm
+
+
+def test_cli_keep_conversations_flag_applies_process_wide(monkeypatch, fixtures_dir):
+    """--keep-conversations lands on the module-level switch, not on the ChatClient kwargs.
+
+    The agentic evaluators build their own clients deep in the call tree, so a constructor
+    kwarg would reach the single-turn path only -- which is the gap --preserve-failed has.
+    """
+    monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok"))
+    applied: list[bool] = []
+
+    class _FakeController:
+        def __init__(self, *a, **k): ...
+        def get_active(self):
+            return ActiveLlmProvider(provider_id="p", default_model_id="gpt-5.2")
+
+        def resolve_and_activate(self, requested, provider=None):
+            return ResolvedModel(provider_id="p", model_id="gpt-5.2", switched=False, provider_name="P")
+
+        def restore(self, original): ...
+        def close(self): ...
+
+    monkeypatch.setattr(cli_main, "WorkspaceModelController", _FakeController)
+    monkeypatch.setattr(cli_main, "keep_conversations", _recording_keep(applied))
+    monkeypatch.setattr(cli_main, "ChatClient", lambda **kwargs: object())
+
+    def _fake_run(items, backend, *, runs, model, workspace_id, **kw):
+        return EvalReport(
+            model=model,
+            workspace_id=workspace_id,
+            items=[
+                ItemReport(id="i1", dataset_name="d", test_kind="visualization", question="q", pass_at_k=True, runs=1)
+            ],
+        )
+
+    monkeypatch.setattr(cli_main, "run_items", _fake_run)
+
+    argv = [
+        "run",
+        "--host",
+        "https://h",
+        "--token",
+        "tok",
+        "--workspace",
+        "ws1",
+        "--dataset",
+        str(fixtures_dir / "sample_dataset"),
+        "--quiet",
+    ]
+    assert cli_main.main([*argv, "--keep-conversations"]) == 0
+    assert applied == [True]
+
+    # Scoped per run, so a bare `run` afterwards asks for nothing. keep=False leaves an
+    # env-set GOODDATA_EVAL_KEEP_CONVERSATIONS alone rather than forcing it off.
+    applied.clear()
+    assert cli_main.main(argv) == 0
+    assert applied == [False]
+
+
 def test_cli_rejects_negative_concurrency(monkeypatch, fixtures_dir):
     monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok"))
     exit_code = cli_main.main(