From 79fdab31d97f948aab5189251dd6b76cd8f83d27 Mon Sep 17 00:00:00 2001
From: nik13 <8347993+nik13@users.noreply.github.com>
Date: Thu, 19 Nov 2026 02:37:56 +0000
Subject: [PATCH 1/8] test(watsonx): cover IBM watsonx.ai text, chat,
streaming, embeddings, error and hidden-content tracing
---
.../watsonx/tests/test_instrumentation.py | 352 ++++++++++++++++++
1 file changed, 352 insertions(+)
create mode 100644 python/frameworks/watsonx/tests/test_instrumentation.py
diff --git a/python/frameworks/watsonx/tests/test_instrumentation.py b/python/frameworks/watsonx/tests/test_instrumentation.py
new file mode 100644
index 000000000..3cf54b23f
--- /dev/null
+++ b/python/frameworks/watsonx/tests/test_instrumentation.py
@@ -0,0 +1,352 @@
+"""Real ibm-watsonx-ai SDK calls with the HTTP transport stubbed: no credentials, no network."""
+import asyncio
+import json
+import socket
+
+import httpx
+import pytest
+import requests
+from fi_instrumentation import TraceConfig
+from ibm_watsonx_ai import APIClient, Credentials
+from ibm_watsonx_ai.foundation_models import ModelInference
+from ibm_watsonx_ai.foundation_models.embeddings import Embeddings
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+from opentelemetry.trace import StatusCode
+
+from traceai_watsonx import WatsonxInstrumentor
+
+LLM = "ibm/granite-3-8b-instruct"
+EMB = "ibm/slate-30m-english-rtrvr"
+SECRET = "sk-synthetic-secret-token"
+
+
+def sse(*events):
+ return "".join(
+ "id: %d\nevent: message\ndata: %s\n\n" % (i + 1, json.dumps(e)) for i, e in enumerate(events)
+ ).encode()
+
+
+class Wire:
+ """Records requests and answers with synthetic watsonx.ai responses."""
+
+ def __init__(self):
+ self.requests = []
+ self.fail_with = None
+
+ def __call__(self, req):
+ url = str(req.url).split("?")[0]
+ self.requests.append(req)
+ if "/v2/projects/" in url:
+ return httpx.Response(200, json={"entity": {"storage": {"type": "bmcos_object_storage"}}, "metadata": {"guid": "p"}})
+ if url.endswith("foundation_model_specs"):
+ return httpx.Response(200, json={"resources": [
+ {"model_id": LLM, "functions": [{"id": "text_generation"}, {"id": "text_chat"}]},
+ {"model_id": EMB, "functions": [{"id": "embedding"}]},
+ ]})
+ if self.fail_with is not None:
+ return httpx.Response(self.fail_with, json={"errors": [{"code": "boom", "message": "leaked " + SECRET}]})
+ if url.endswith("text/generation_stream"):
+ return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=sse(
+ {"model_id": LLM, "results": [{"generated_text": "He", "generated_token_count": 1, "input_token_count": 5, "stop_reason": "not_finished"}]},
+ {"model_id": LLM, "results": [{"generated_text": "llo", "generated_token_count": 3, "input_token_count": 0, "stop_reason": "eos_token"}]},
+ ))
+ if url.endswith("text/chat_stream"):
+ return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=sse(
+ {"id": "c1", "model_id": LLM, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "yo"}, "finish_reason": None}]},
+ {"id": "c1", "model_id": LLM, "choices": [{"index": 0, "delta": {"content": "!"}, "finish_reason": "stop"}],
+ "usage": {"prompt_tokens": 4, "completion_tokens": 2, "total_tokens": 6}},
+ ))
+ if url.endswith("text/generation"):
+ return httpx.Response(200, json={"model_id": LLM, "results": [
+ {"generated_text": "Tokyo", "generated_token_count": 3, "input_token_count": 5, "stop_reason": "eos_token"}]})
+ if url.endswith("text/chat"):
+ return httpx.Response(200, json={"id": "c1", "model_id": LLM, "choices": [
+ {"index": 0, "message": {"role": "assistant", "content": "Tokyo"}, "finish_reason": "stop"}],
+ "usage": {"prompt_tokens": 4, "completion_tokens": 2, "total_tokens": 6}})
+ if url.endswith("text/embeddings"):
+ n = len(json.loads(req.content)["inputs"])
+ return httpx.Response(200, json={"model_id": EMB, "input_token_count": 7,
+ "results": [{"embedding": [0.25, 0.5, 0.75]}] * n})
+ return httpx.Response(404, json={"errors": [{"code": "nf", "message": url}]})
+
+
+def _as_requests_response(response, request):
+ out = requests.Response()
+ out.status_code = response.status_code
+ out._content = response.content
+ out.headers.update(response.headers)
+ out.url = request.url
+ out.request = request
+ return out
+
+
+@pytest.fixture
+def wire(monkeypatch):
+ """Route every HTTP library the SDK uses (httpx, and requests in older SDKs) to the stub."""
+ wire = Wire()
+ for cls in (httpx.Client, httpx.AsyncClient):
+ original = cls.__init__
+
+ def init(self, *a, __orig=original, **k):
+ k["transport"] = httpx.MockTransport(wire)
+ __orig(self, *a, **k)
+
+ monkeypatch.setattr(cls, "__init__", init)
+
+ def send(self, request, **kwargs):
+ body = request.body.encode() if isinstance(request.body, str) else (request.body or b"")
+ stubbed = wire(httpx.Request(request.method, request.url, content=body))
+ return _as_requests_response(stubbed, request)
+
+ monkeypatch.setattr(requests.adapters.HTTPAdapter, "send", send)
+
+ def no_network(*a, **k):
+ raise AssertionError("network access attempted")
+
+ monkeypatch.setattr(socket.socket, "connect", no_network)
+ return wire
+
+
+@pytest.fixture
+def api(wire):
+ return APIClient(credentials=Credentials(url="https://us-south.ml.cloud.ibm.com", token=SECRET), project_id="p")
+
+
+def make_spans(config=None):
+ exporter = InMemorySpanExporter()
+ provider = TracerProvider()
+ provider.add_span_processor(SimpleSpanProcessor(exporter))
+ instrumentor = WatsonxInstrumentor()
+ instrumentor.instrument(tracer_provider=provider, **({"config": config} if config else {}))
+ return exporter, provider, instrumentor
+
+
+@pytest.fixture
+def spans():
+ exporter, provider, instrumentor = make_spans()
+ yield exporter
+ instrumentor.uninstrument()
+ provider.shutdown()
+
+
+@pytest.fixture
+def model(api):
+ return ModelInference(model_id=LLM, api_client=api)
+
+
+def one(spans):
+ finished = spans.get_finished_spans()
+ assert len(finished) == 1, [s.name for s in finished]
+ return finished[0], dict(finished[0].attributes)
+
+
+def test_generate_records_llm_span(spans, model):
+ result = model.generate(prompt="Capital of Japan?", params={"max_new_tokens": 20, "temperature": 0.5, "top_p": 0.9})
+ assert result["results"][0]["generated_text"] == "Tokyo"
+ span, a = one(spans)
+ assert a["gen_ai.span.kind"] == "LLM"
+ assert a["gen_ai.provider.name"] == "ibm.watsonx.ai"
+ assert a["gen_ai.operation.name"] == "text_completion"
+ assert a["gen_ai.request.model"] == LLM
+ assert a["gen_ai.request.max_tokens"] == 20
+ assert a["gen_ai.request.temperature"] == 0.5
+ assert a["gen_ai.request.top_p"] == 0.9
+ assert json.loads(a["gen_ai.request.parameters"])["max_new_tokens"] == 20
+ assert a["gen_ai.response.model"] == LLM
+ assert a["gen_ai.usage.input_tokens"] == 5
+ assert a["gen_ai.usage.output_tokens"] == 3
+ assert tuple(a["gen_ai.response.finish_reasons"]) == ("eos_token",)
+ assert a["gen_ai.input.messages.0.message.role"] == "user"
+ assert a["gen_ai.input.messages.0.message.content"] == "Capital of Japan?"
+ assert a["gen_ai.output.messages.0.message.role"] == "assistant"
+ assert a["gen_ai.output.messages.0.message.content"] == "Tokyo"
+ assert span.status.status_code != StatusCode.ERROR
+
+
+def test_generate_text_makes_exactly_one_span(spans, model):
+ assert model.generate_text(prompt="Capital of Japan?") == "Tokyo"
+ span, a = one(spans) # generate_text calls generate internally: no nested duplicate
+ assert a["gen_ai.output.messages.0.message.content"] == "Tokyo"
+
+
+def test_generate_text_parameter_object(spans, model):
+ from ibm_watsonx_ai.foundation_models.schema import TextGenParameters
+
+ model.generate_text(prompt="x", params=TextGenParameters(max_new_tokens=7, temperature=0.1))
+ _, a = one(spans)
+ assert a["gen_ai.request.max_tokens"] == 7
+ assert a["gen_ai.request.temperature"] == 0.1
+
+
+def test_chat_records_messages_usage_and_finish(spans, model):
+ messages = [{"role": "system", "content": "Be brief."}, {"role": "user", "content": "Capital of Japan?"}]
+ out = model.chat(messages=messages, params={"max_tokens": 9, "temperature": 0.2})
+ assert out["choices"][0]["message"]["content"] == "Tokyo"
+ _, a = one(spans)
+ assert a["gen_ai.span.kind"] == "LLM"
+ assert a["gen_ai.operation.name"] == "chat"
+ assert a["gen_ai.request.model"] == LLM
+ assert a["gen_ai.request.max_tokens"] == 9
+ assert a["gen_ai.input.messages.0.message.role"] == "system"
+ assert a["gen_ai.input.messages.1.message.content"] == "Capital of Japan?"
+ assert a["gen_ai.output.messages.0.message.content"] == "Tokyo"
+ assert a["gen_ai.response.id"] == "c1"
+ assert a["gen_ai.usage.input_tokens"] == 4
+ assert a["gen_ai.usage.output_tokens"] == 2
+ assert a["gen_ai.usage.total_tokens"] == 6
+ assert tuple(a["gen_ai.response.finish_reasons"]) == ("stop",)
+
+
+def test_generate_text_stream_assembles_output_and_ends_on_exhaustion(spans, model):
+ stream = model.generate_text_stream(prompt="Hi")
+ assert spans.get_finished_spans() == ()
+ assert list(stream) == ["He", "llo"]
+ _, a = one(spans)
+ assert a["gen_ai.output.messages.0.message.content"] == "Hello"
+ assert "gen_ai.usage.input_tokens" not in a # text-only chunks carry no counts
+ assert a["gen_ai.input.messages.0.message.content"] == "Hi"
+
+
+def test_generate_text_stream_raw_chunks_pass_through(spans, model):
+ chunks = list(model.generate_text_stream(prompt="Hi", raw_response=True))
+ assert [c["results"][0]["generated_text"] for c in chunks] == ["He", "llo"]
+ _, a = one(spans)
+ assert a["gen_ai.output.messages.0.message.content"] == "Hello"
+ assert a["gen_ai.usage.input_tokens"] == 5
+ assert a["gen_ai.usage.output_tokens"] == 3
+ assert tuple(a["gen_ai.response.finish_reasons"]) == ("eos_token",)
+
+
+def test_chat_stream_assembles_output_usage(spans, model):
+ chunks = list(model.chat_stream(messages=[{"role": "user", "content": "hi"}]))
+ assert len(chunks) == 2
+ _, a = one(spans)
+ assert a["gen_ai.output.messages.0.message.content"] == "yo!"
+ assert a["gen_ai.output.messages.0.message.role"] == "assistant"
+ assert a["gen_ai.usage.input_tokens"] == 4
+ assert a["gen_ai.usage.output_tokens"] == 2
+ assert tuple(a["gen_ai.response.finish_reasons"]) == ("stop",)
+
+
+def test_abandoned_stream_ends_span_on_close_without_output(spans, model):
+ stream = model.chat_stream(messages=[{"role": "user", "content": "hi"}])
+ next(stream)
+ assert spans.get_finished_spans() == ()
+ stream.close()
+ span, a = one(spans)
+ assert a["watsonx.stream.incomplete"] is True
+ assert "gen_ai.output.messages.0.message.content" not in a
+ assert span.status.status_code != StatusCode.ERROR
+
+
+def test_embeddings_documents_and_query(spans, api):
+ emb = Embeddings(model_id=EMB, api_client=api)
+ assert emb.embed_documents(["alpha", "beta"]) == [[0.25, 0.5, 0.75]] * 2
+ assert emb.embed_query("gamma") == [0.25, 0.5, 0.75]
+ finished = spans.get_finished_spans()
+ assert len(finished) == 2 # embed_* call generate internally: no nested duplicates
+ docs, query = (dict(s.attributes) for s in finished)
+ assert docs["gen_ai.span.kind"] == "EMBEDDING"
+ assert docs["gen_ai.provider.name"] == "ibm.watsonx.ai"
+ assert docs["gen_ai.operation.name"] == "embeddings"
+ assert docs["gen_ai.request.model"] == EMB
+ assert docs["embedding.embeddings.0.embedding.text"] == "alpha"
+ assert docs["embedding.embeddings.1.embedding.text"] == "beta"
+ assert docs["gen_ai.embeddings.dimension.count"] == 3
+ assert query["embedding.embeddings.0.embedding.text"] == "gamma"
+ for attrs in (docs, query): # vectors are never exported by default
+ assert not any("vector" in k for k in attrs)
+ assert "0.25" not in json.dumps(attrs, default=str)
+
+
+def test_error_sets_status_error_type_without_message_and_reraises(spans, model, wire):
+ from ibm_watsonx_ai.wml_client_error import ApiRequestFailure
+
+ wire.fail_with = 500
+ with pytest.raises(ApiRequestFailure):
+ model.chat(messages=[{"role": "user", "content": "hi"}])
+ span, a = one(spans)
+ assert span.status.status_code == StatusCode.ERROR
+ assert a["error.type"] == "ApiRequestFailure"
+ assert SECRET not in json.dumps(a, default=str)
+ assert not span.events # no recorded exception message or stack
+
+
+def test_stream_error_ends_span_with_error(spans, model, wire):
+ from ibm_watsonx_ai.wml_client_error import WMLClientError
+
+ wire.fail_with = 500
+ with pytest.raises(WMLClientError):
+ list(model.generate_text_stream(prompt="Hi"))
+ span, a = one(spans)
+ assert span.status.status_code == StatusCode.ERROR
+ assert a["error.type"] == "WMLClientError"
+ assert SECRET not in json.dumps(a, default=str)
+
+
+def test_async_variants(spans, model, api):
+ emb = Embeddings(model_id=EMB, api_client=api)
+ async_embeddings = hasattr(Embeddings, "aembed_query") # absent in the oldest supported SDK
+
+ async def run():
+ chat = await model.achat(messages=[{"role": "user", "content": "hi"}])
+ stream = await model.achat_stream(messages=[{"role": "user", "content": "hi"}])
+ chunks = [c async for c in stream]
+ gen = await model.agenerate(prompt="x")
+ gstream = await model.agenerate_stream(prompt="x")
+ gchunks = [c async for c in gstream]
+ vectors = (await emb.aembed_query("q"), await emb.aembed_documents(["a"])) if async_embeddings else None
+ return chat, chunks, gen, gchunks, vectors
+
+ chat, chunks, gen, gchunks, vectors = asyncio.run(run())
+ assert chat["choices"][0]["message"]["content"] == "Tokyo" and len(chunks) == 2 and len(gchunks) == 2
+ assert gen["results"][0]["generated_text"] == "Tokyo"
+ finished = [dict(s.attributes) for s in spans.get_finished_spans()]
+ assert len(finished) == (6 if async_embeddings else 4), [a["gen_ai.operation.name"] for a in finished]
+ if async_embeddings:
+ assert vectors == ([0.25, 0.5, 0.75], [[0.25, 0.5, 0.75]])
+ assert sum(a["gen_ai.span.kind"] == "EMBEDDING" for a in finished) == 2
+ outs = sorted(
+ a["gen_ai.output.messages.0.message.content"] for a in finished
+ if "gen_ai.output.messages.0.message.content" in a
+ )
+ assert outs == ["Hello", "Tokyo", "Tokyo", "yo!"]
+
+
+def test_hidden_content_removes_prompts_outputs_but_keeps_metadata(api, wire):
+ exporter, provider, instrumentor = make_spans(TraceConfig(hide_inputs=True, hide_outputs=True,
+ hide_input_messages=True, hide_output_messages=True,
+ hide_input_text=True, hide_output_text=True))
+ try:
+ model = ModelInference(model_id=LLM, api_client=api)
+ model.chat(messages=[{"role": "user", "content": "my private question"}])
+ list(model.generate_text_stream(prompt="my private prompt"))
+ Embeddings(model_id=EMB, api_client=api).embed_query("my private text")
+ for span in exporter.get_finished_spans():
+ dumped = json.dumps(dict(span.attributes), default=str)
+ assert "private" not in dumped and "Tokyo" not in dumped and "Hello" not in dumped and "yo!" not in dumped
+ assert span.attributes["gen_ai.request.model"] in (LLM, EMB)
+ chat = [s for s in exporter.get_finished_spans() if s.attributes.get("gen_ai.operation.name") == "chat"][0]
+ assert chat.attributes["gen_ai.usage.input_tokens"] == 4
+ finally:
+ instrumentor.uninstrument()
+ provider.shutdown()
+
+
+def test_credentials_never_exported(spans, model):
+ model.chat(messages=[{"role": "user", "content": "hi"}])
+ _, a = one(spans)
+ assert SECRET not in json.dumps(a, default=str)
+
+
+def test_uninstrument_restores_originals_and_stops_tracing(api):
+ before = ModelInference.chat
+ exporter, provider, instrumentor = make_spans()
+ assert ModelInference.chat is not before
+ instrumentor.uninstrument()
+ provider.shutdown()
+ assert ModelInference.chat is before
+ ModelInference(model_id=LLM, api_client=api).chat(messages=[{"role": "user", "content": "hi"}])
+ assert exporter.get_finished_spans() == ()
From 26845c674bdc2b48cec5071b3d8f75bd7f202a56 Mon Sep 17 00:00:00 2001
From: nik13 <8347993+nik13@users.noreply.github.com>
Date: Thu, 19 Nov 2026 02:42:09 +0000
Subject: [PATCH 2/8] feat(watsonx): trace IBM watsonx.ai inference and
embeddings, with zero-code registration
Add traceAI-watsonx (traceai_watsonx.WatsonxInstrumentor) for ibm-watsonx-ai 1.3.0 up to 2.0:
ModelInference generate/generate_text/generate_text_stream/chat/chat_stream and their async
variants, and Embeddings generate/embed_documents/embed_query and async variants. Register
the integration for zero-code instrumentation and wire release automation, CI and docs.
---
.github/assets/integration-coverage.json | 7 +
.github/release/config.json | 3 +-
.github/workflows/watsonx-python-tests.yml | 65 +++
.release-please-manifest.json | 1 +
README.md | 2 +-
docs/python/auto-instrumentation.md | 1 +
python/README.md | 1 +
python/fi_instrumentation/auto/_registry.py | 1 +
python/frameworks/watsonx/CHANGELOG.md | 5 +
python/frameworks/watsonx/README.md | 46 ++
python/frameworks/watsonx/pyproject.toml | 26 ++
.../watsonx/traceai_watsonx/__init__.py | 432 ++++++++++++++++++
.../watsonx/traceai_watsonx/version.py | 1 +
python/tests/test_auto_entrypoints.py | 1 +
release-please-config.json | 8 +
15 files changed, 598 insertions(+), 2 deletions(-)
create mode 100644 .github/workflows/watsonx-python-tests.yml
create mode 100644 python/frameworks/watsonx/CHANGELOG.md
create mode 100644 python/frameworks/watsonx/README.md
create mode 100644 python/frameworks/watsonx/pyproject.toml
create mode 100644 python/frameworks/watsonx/traceai_watsonx/__init__.py
create mode 100644 python/frameworks/watsonx/traceai_watsonx/version.py
diff --git a/.github/assets/integration-coverage.json b/.github/assets/integration-coverage.json
index bbaf64906..e5fcc77e6 100644
--- a/.github/assets/integration-coverage.json
+++ b/.github/assets/integration-coverage.json
@@ -453,6 +453,13 @@
"name": "IBM watsonx",
"category": "model-providers",
"languages": {
+ "Python": [
+ {
+ "manifest": "python/frameworks/watsonx/pyproject.toml",
+ "package": "traceAI-watsonx",
+ "source": "python/frameworks/watsonx/"
+ }
+ ],
"Java": [
{
"manifest": "java/traceai-java-watsonx/pom.xml",
diff --git a/.github/release/config.json b/.github/release/config.json
index 32fafb62d..7e97bf100 100644
--- a/.github/release/config.json
+++ b/.github/release/config.json
@@ -29,7 +29,8 @@
"traceAI-transformers",
"traceAI-openrouter",
"traceAI-sagemaker",
- "traceAI-marqo"
+ "traceAI-marqo",
+ "traceAI-watsonx"
],
"_hold_artifacts_comment": "Reactor modules that the Java release builds and tests but does not deploy, with the reason. A held module's release profile sets skipPublishing, and the release plan leaves it out of the Central check, so the rest of the reactor still releases. To publish one, remove its entry here and its skipPublishing setting.",
"hold_artifacts": {}
diff --git a/.github/workflows/watsonx-python-tests.yml b/.github/workflows/watsonx-python-tests.yml
new file mode 100644
index 000000000..04e97bdd6
--- /dev/null
+++ b/.github/workflows/watsonx-python-tests.yml
@@ -0,0 +1,65 @@
+name: IBM watsonx.ai Python SDK contracts
+
+on:
+ pull_request:
+ branches: [main, dev]
+ paths:
+ - 'python/frameworks/watsonx/**'
+ - 'python/fi_instrumentation/**'
+ - '.github/workflows/watsonx-python-tests.yml'
+ push:
+ branches: [dev]
+ paths:
+ - 'python/frameworks/watsonx/**'
+ - 'python/fi_instrumentation/**'
+ - '.github/workflows/watsonx-python-tests.yml'
+
+permissions:
+ contents: read
+
+jobs:
+ sdk-contract:
+ runs-on: ubuntu-24.04
+ timeout-minutes: 15
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - python: '3.10'
+ watsonx: '1.3.0'
+ artifact: wheel
+ - python: '3.13'
+ watsonx: '1.9.1'
+ artifact: sdist
+ steps:
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+ with:
+ persist-credentials: false
+ - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ with:
+ python-version: ${{ matrix.python }}
+ - name: Build and install packages
+ shell: bash
+ env:
+ ARTIFACT: ${{ matrix.artifact }}
+ WATSONX_VERSION: ${{ matrix.watsonx }}
+ run: |
+ python -m venv .watsonx-test-venv
+ .watsonx-test-venv/bin/pip install build
+ .watsonx-test-venv/bin/python -m build "--$ARTIFACT" python/frameworks/watsonx
+ if [ "$ARTIFACT" = wheel ]; then
+ packages=(python/frameworks/watsonx/dist/*.whl)
+ else
+ packages=(python/frameworks/watsonx/dist/*.tar.gz)
+ fi
+ .watsonx-test-venv/bin/pip install "${packages[@]}" \
+ "ibm-watsonx-ai==$WATSONX_VERSION" pytest pytest-asyncio
+ - name: Run SDK tests without external traffic
+ shell: bash
+ run: |
+ sudo unshare --net bash -c '
+ set -e
+ ip link set lo up
+ "$1" -I -m pytest python/frameworks/watsonx/tests \
+ -q -p no:cacheprovider
+ ' _ "$PWD/.watsonx-test-venv/bin/python"
diff --git a/.release-please-manifest.json b/.release-please-manifest.json
index e3ccf0460..98469dcfc 100644
--- a/.release-please-manifest.json
+++ b/.release-please-manifest.json
@@ -160,6 +160,7 @@
"python/frameworks/openrouter": "0.1.0",
"python/frameworks/sagemaker": "0.1.0",
"python/frameworks/marqo": "0.1.0",
+ "python/frameworks/watsonx": "0.1.0",
"go/langchaingo": "0.1.0",
"ruby": "0.1.0",
"ruby/roast": "0.1.0",
diff --git a/README.md b/README.md
index d1b04f9a0..451710e05 100644
--- a/README.md
+++ b/README.md
@@ -340,7 +340,7 @@ Hosted and local model inference, embeddings, and reranking.
|
Google GenAI (Gemini) | [SDK](python/frameworks/google-genai/ "traceAI-google-genai") · [Live](python/frameworks/google-genai-live/ "traceAI-google-genai-live") | [Legacy SDK only](typescript/packages/traceai_google_genai/ "@google/generative-ai compatibility; not @google/genai") | [SDK](java/traceai-java-google-genai/ "traceai-java-google-genai") | [SDK](go/genai/ "github.com/future-agi/traceAI/go/genai") | — |
|
Groq | [SDK](python/frameworks/groq/ "traceAI-groq") | [SDK](typescript/packages/traceai_groq/ "@traceai/groq") | — | — | — |
|
Hugging Face | [SDK](python/frameworks/huggingface/ "traceAI-huggingface") | [SDK](typescript/packages/traceai_huggingface/ "@traceai/huggingface") | — | — | — |
-|
IBM watsonx | — | — | [SDK](java/traceai-java-watsonx/ "traceai-java-watsonx") | — | — |
+|
IBM watsonx | [SDK](python/frameworks/watsonx/ "traceAI-watsonx") | — | [SDK](java/traceai-java-watsonx/ "traceai-java-watsonx") | — | — |
|
MiniMax | [SDK](python/frameworks/minimax/ "traceAI-minimax") | — | — | — | — |
|
Mistral AI | [SDK](python/frameworks/mistralai/ "traceAI-mistralai") | [SDK](typescript/packages/traceai_mistral/ "@traceai/mistral") | — | — | — |
|
Ollama | [SDK](python/frameworks/ollama/ "traceAI-ollama") | [SDK](typescript/packages/traceai_ollama/ "@traceai/ollama") | [SDK](java/traceai-java-ollama/ "traceai-java-ollama") | — | — |
diff --git a/docs/python/auto-instrumentation.md b/docs/python/auto-instrumentation.md
index b72260c91..1350b1a78 100644
--- a/docs/python/auto-instrumentation.md
+++ b/docs/python/auto-instrumentation.md
@@ -99,6 +99,7 @@ Each integration reports one of these statuses:
| `litellm` | `litellm` | `traceAI-litellm` | Automatic |
| `openrouter` | `openrouter` | `traceAI-openrouter` | Automatic |
| `portkey` | `portkey-ai` | `traceAI-portkey` | Automatic |
+| `watsonx` | `ibm_watsonx_ai` | `traceAI-watsonx` | Automatic |
| `replicate` | `replicate` | `traceAI-replicate` | Automatic |
| `voyage` | `voyageai` | `traceAI-voyage` | Automatic |
| `discoveryengine` | `google-cloud-discoveryengine` | `traceAI-discoveryengine` | Automatic |
diff --git a/python/README.md b/python/README.md
index 6b7cd68a3..874c04059 100644
--- a/python/README.md
+++ b/python/README.md
@@ -212,6 +212,7 @@ Each package below links to its own installation instructions, supported SDK ver
| `traceAI-redis` | [redis-vector](frameworks/redis-vector/) |
| `traceAI-replicate` | [replicate](frameworks/replicate/) |
| `traceAI-sagemaker` | [sagemaker](frameworks/sagemaker/) |
+| `traceAI-watsonx` | [watsonx](frameworks/watsonx/) |
| `traceAI-semantic-kernel` | [semantic-kernel](frameworks/semantic-kernel/) |
| `traceAI-smolagents` | [smolagents](frameworks/smolagents/) |
| `traceai-strands` | [strands](frameworks/strands/) |
diff --git a/python/fi_instrumentation/auto/_registry.py b/python/fi_instrumentation/auto/_registry.py
index 53bea8096..f893db9a9 100644
--- a/python/fi_instrumentation/auto/_registry.py
+++ b/python/fi_instrumentation/auto/_registry.py
@@ -69,6 +69,7 @@ def _covered_by_openai(provider: str) -> str:
Integration("litellm", ("litellm",), "traceAI-litellm", "traceai_litellm", "LiteLLMInstrumentor", category="gateway"),
Integration("openrouter", ("openrouter",), "traceAI-openrouter", "traceai_openrouter", "OpenRouterInstrumentor", category="gateway"),
Integration("portkey", ("portkey_ai",), "traceAI-portkey", "traceai_portkey", "PortkeyInstrumentor", category="gateway"),
+ Integration("watsonx", ("ibm_watsonx_ai",), "traceAI-watsonx", "traceai_watsonx", "WatsonxInstrumentor"),
Integration("replicate", ("replicate",), "traceAI-replicate", "traceai_replicate", "ReplicateInstrumentor"),
Integration("voyage", ("voyageai",), "traceAI-voyage", "traceai_voyage", "VoyageInstrumentor"),
Integration("discoveryengine", ("google.cloud.discoveryengine",), "traceAI-discoveryengine", "traceai_discoveryengine", "DiscoveryEngineInstrumentor", category="retrieval"),
diff --git a/python/frameworks/watsonx/CHANGELOG.md b/python/frameworks/watsonx/CHANGELOG.md
new file mode 100644
index 000000000..888720186
--- /dev/null
+++ b/python/frameworks/watsonx/CHANGELOG.md
@@ -0,0 +1,5 @@
+# Changelog
+
+## 0.1.0
+
+- Instrument the IBM watsonx.ai Python SDK (`ibm-watsonx-ai`): `ModelInference` text generation, chat and their streaming and asynchronous variants, and `Embeddings`.
diff --git a/python/frameworks/watsonx/README.md b/python/frameworks/watsonx/README.md
new file mode 100644
index 000000000..9500f7b90
--- /dev/null
+++ b/python/frameworks/watsonx/README.md
@@ -0,0 +1,46 @@
+# IBM watsonx.ai Python SDK tracing
+
+Trace calls made through the [`ibm-watsonx-ai` Python SDK](https://pypi.org/project/ibm-watsonx-ai/) with the maintained `traceAI-watsonx` package. It supports `ibm-watsonx-ai` 1.3.0 up to, but not including, 2.0.
+
+```bash
+pip install traceAI-watsonx
+```
+
+Set `FI_API_KEY`, `FI_SECRET_KEY`, and your watsonx.ai credentials (`WATSONX_APIKEY`, `WATSONX_URL`, `WATSONX_PROJECT_ID`) in your environment. Then:
+
+```python
+import os
+
+from fi_instrumentation import register
+from fi_instrumentation.fi_types import ProjectType
+from ibm_watsonx_ai import Credentials
+from ibm_watsonx_ai.foundation_models import ModelInference
+from traceai_watsonx import WatsonxInstrumentor
+
+provider = register(project_type=ProjectType.OBSERVE, project_name="watsonx-demo")
+WatsonxInstrumentor().instrument(tracer_provider=provider)
+model = ModelInference(
+ model_id="ibm/granite-3-8b-instruct",
+ credentials=Credentials(api_key=os.environ["WATSONX_APIKEY"], url=os.environ["WATSONX_URL"]),
+ project_id=os.environ["WATSONX_PROJECT_ID"],
+)
+answer = model.chat(messages=[{"role": "user", "content": "What is the capital of Japan?"}])
+print(answer["choices"][0]["message"]["content"])
+provider.force_flush()
+```
+
+With the `fi-instrument` launcher or `FI_INSTRUMENTATIONS`, no code is needed: importing `ibm_watsonx_ai` turns this integration on.
+
+Find the `watsonx.chat` span in your Future AGI project. Covered calls:
+
+| Span | SDK methods |
+| --- | --- |
+| `watsonx.text_completion` | `ModelInference.generate`, `generate_text`, `generate_text_stream`, `agenerate`, `agenerate_stream` |
+| `watsonx.chat` | `ModelInference.chat`, `chat_stream`, `achat`, `achat_stream` |
+| `watsonx.embeddings` | `Embeddings.generate`, `embed_documents`, `embed_query`, `agenerate`, `aembed_documents`, `aembed_query` |
+
+Each span records `gen_ai.provider.name` (`ibm.watsonx.ai`), the request model or deployment id, request parameters, input and output messages, token usage (from `input_token_count` / `generated_token_count` or the chat `usage`), and finish reasons. `generate_text` returns only the text, so its span has no token counts; use `generate` or `generate_text(raw_response=True)` to get them. One SDK call is one span: methods that call each other inside the SDK (`generate_text` calls `generate`, `embed_query` calls `generate`) are not traced twice.
+
+Streams finish when fully consumed or closed; a stream abandoned without closing it is marked `watsonx.stream.incomplete` and carries no partial output. Errors set the span status to ERROR and `error.type`, and the exception is re-raised unchanged. Exception messages, stacks, credentials and embedding vectors are never exported. Non-text parts of chat messages (images) are not captured.
+
+Pass `TraceConfig(hide_inputs=True, hide_outputs=True)` to `instrument(config=...)` to remove prompt and response content. Do not instrument HTTPX for the same requests, or you will get duplicate spans. The package does not send requests to IBM on its own.
diff --git a/python/frameworks/watsonx/pyproject.toml b/python/frameworks/watsonx/pyproject.toml
new file mode 100644
index 000000000..41430bd45
--- /dev/null
+++ b/python/frameworks/watsonx/pyproject.toml
@@ -0,0 +1,26 @@
+[tool.poetry]
+name = "traceAI-watsonx"
+version = "0.1.0"
+description = "OpenTelemetry instrumentation for IBM watsonx.ai (ibm-watsonx-ai) Python SDK"
+authors = ["Future AGI "]
+readme = "README.md"
+license = "Apache-2.0"
+packages = [{ include = "traceai_watsonx" }]
+classifiers = ["Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Framework :: OpenTelemetry :: Instrumentations"]
+
+[tool.poetry.dependencies]
+python = ">=3.10,<3.14"
+fi-instrumentation-otel = ">=1.1.0"
+ibm-watsonx-ai = ">=1.3.0,<2"
+wrapt = ">=1.15.0"
+
+[tool.poetry.group.dev.dependencies]
+pytest = ">=7"
+pytest-asyncio = ">=0.23"
+
+[tool.poetry.urls]
+Repository = "https://github.com/future-agi/traceAI/tree/dev/python/frameworks/watsonx"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
diff --git a/python/frameworks/watsonx/traceai_watsonx/__init__.py b/python/frameworks/watsonx/traceai_watsonx/__init__.py
new file mode 100644
index 000000000..6702a1c97
--- /dev/null
+++ b/python/frameworks/watsonx/traceai_watsonx/__init__.py
@@ -0,0 +1,432 @@
+"""Trace the IBM ``ibm-watsonx-ai`` Python SDK (``ModelInference`` and ``Embeddings``).
+
+Spans are created at the public SDK methods. The SDK's own methods call each other
+(``generate_text`` -> ``generate``, ``embed_query`` -> ``generate``); only the outermost call is
+traced, so one user call is one span. Do not also instrument HTTPX for the same requests.
+"""
+from __future__ import annotations
+
+import contextvars
+import json
+from importlib import import_module
+from typing import Any, Collection, Optional
+
+from fi_instrumentation import FITracer, TraceConfig
+from fi_instrumentation.fi_types import FiSpanKindValues, SpanAttributes as A
+from opentelemetry import trace
+from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
+from opentelemetry.trace import Status, StatusCode
+from wrapt import wrap_function_wrapper
+
+from .version import __version__
+
+PROVIDER = "ibm.watsonx.ai"
+_LIMIT = 32768
+_INFERENCE = "ibm_watsonx_ai.foundation_models.inference.model_inference"
+_EMBEDDINGS = "ibm_watsonx_ai.foundation_models.embeddings.embeddings"
+
+# (module, class, method, operation, is_async)
+_METHODS = (
+ (_INFERENCE, "ModelInference", "generate", "text_completion", False),
+ (_INFERENCE, "ModelInference", "generate_text", "text_completion", False),
+ (_INFERENCE, "ModelInference", "generate_text_stream", "text_completion", False),
+ (_INFERENCE, "ModelInference", "agenerate", "text_completion", True),
+ (_INFERENCE, "ModelInference", "agenerate_stream", "text_completion", True),
+ (_INFERENCE, "ModelInference", "chat", "chat", False),
+ (_INFERENCE, "ModelInference", "chat_stream", "chat", False),
+ (_INFERENCE, "ModelInference", "achat", "chat", True),
+ (_INFERENCE, "ModelInference", "achat_stream", "chat", True),
+ (_EMBEDDINGS, "Embeddings", "generate", "embeddings", False),
+ (_EMBEDDINGS, "Embeddings", "embed_documents", "embeddings", False),
+ (_EMBEDDINGS, "Embeddings", "embed_query", "embeddings", False),
+ (_EMBEDDINGS, "Embeddings", "agenerate", "embeddings", True),
+ (_EMBEDDINGS, "Embeddings", "aembed_documents", "embeddings", True),
+ (_EMBEDDINGS, "Embeddings", "aembed_query", "embeddings", True),
+)
+
+# True while a traced SDK call is running, so the SDK's internal calls to other public methods
+# do not create nested duplicate spans.
+_INSIDE: contextvars.ContextVar[bool] = contextvars.ContextVar("traceai_watsonx_inside", default=False)
+
+_REQUEST_ATTRIBUTES = (
+ (A.GEN_AI_REQUEST_MAX_TOKENS, ("max_new_tokens", "max_tokens", "max_completion_tokens"), int),
+ (A.GEN_AI_REQUEST_TEMPERATURE, ("temperature",), float),
+ (A.GEN_AI_REQUEST_TOP_P, ("top_p",), float),
+ (A.GEN_AI_REQUEST_TOP_K, ("top_k",), int),
+ (A.GEN_AI_REQUEST_FREQUENCY_PENALTY, ("frequency_penalty",), float),
+ (A.GEN_AI_REQUEST_PRESENCE_PENALTY, ("presence_penalty",), float),
+ (A.GEN_AI_REQUEST_SEED, ("random_seed", "seed"), int),
+)
+
+
+def _field(value: Any, name: str) -> Any:
+ if isinstance(value, dict):
+ return value.get(name)
+ return getattr(value, name, None)
+
+
+def _number(value: Any, kind: type = int) -> Optional[Any]:
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
+ return None
+ if kind is int:
+ return int(value) if float(value).is_integer() and value >= 0 else None
+ return float(value)
+
+
+def _argument(args: tuple, kwargs: dict, index: int, name: str) -> Any:
+ if name in kwargs:
+ return kwargs[name]
+ return args[index] if len(args) > index else None
+
+
+def _params(value: Any) -> dict:
+ """Request parameters as a plain dict, from a dict or an SDK parameter object."""
+ if hasattr(value, "to_dict"):
+ try:
+ value = value.to_dict()
+ except Exception:
+ return {}
+ return dict(value) if isinstance(value, dict) else {}
+
+
+def _json(value: Any) -> str:
+ return json.dumps(value, default=str, sort_keys=True)
+
+
+def _text_content(content: Any) -> Optional[str]:
+ """Plain text of a message; image and other non-text parts are never captured."""
+ if isinstance(content, str):
+ return content[:_LIMIT]
+ if isinstance(content, (list, tuple)):
+ parts = [p.get("text") for p in content if isinstance(p, dict) and p.get("type") == "text"]
+ parts = [p for p in parts if isinstance(p, str)]
+ return "".join(parts)[:_LIMIT] if parts else None
+ return None
+
+
+def _failure(span: Any, exc: BaseException) -> None:
+ # Exception text may carry the request body, URL or credentials: record only the type.
+ span.set_attribute("error.type", type(exc).__name__)
+ span.set_status(Status(StatusCode.ERROR))
+ span.end()
+
+
+class _Output:
+ """Accumulates response (or stream chunk) content, usage and finish reasons."""
+
+ def __init__(self) -> None:
+ self.text: dict[int, list[str]] = {}
+ self.role: dict[int, str] = {}
+ self.finish: dict[int, str] = {}
+ self.model: Optional[str] = None
+ self.id: Optional[str] = None
+ self.tokens_in: Optional[int] = None
+ self.tokens_out: Optional[int] = None
+ self.tokens_total: Optional[int] = None
+ self.vectors = 0
+ self.dimension: Optional[int] = None
+ self.size = 0
+
+ def _add(self, index: int, piece: Any) -> None:
+ if isinstance(piece, str) and piece and self.size < _LIMIT:
+ piece = piece[: _LIMIT - self.size]
+ self.text.setdefault(index, []).append(piece)
+ self.size += len(piece)
+
+ def _tokens(self, attr: str, value: Any, total: bool) -> None:
+ number = _number(value)
+ if number is None:
+ return
+ current = getattr(self, attr)
+ # Whole responses add up; stream chunks carry running counts, so keep the largest.
+ setattr(self, attr, number if current is None else (current + number if total else max(current, number)))
+
+ def feed(self, chunk: Any, offset: int = 0, total: bool = False) -> None:
+ if isinstance(chunk, str):
+ self._add(offset, chunk)
+ elif isinstance(chunk, (list, tuple)) and chunk and all(_number(x, float) is not None for x in chunk):
+ self.dimension = len(chunk) # embed_query: one vector
+ elif isinstance(chunk, (list, tuple)) and chunk and isinstance(chunk[0], (list, tuple)):
+ first = chunk[0] # embed_documents: a list of vectors
+ if first and all(_number(x, float) is not None for x in first):
+ self.dimension = len(first)
+ elif isinstance(chunk, (list, tuple)): # generate() over a list of prompts
+ for position, item in enumerate(chunk):
+ self.feed(item, offset=offset + position, total=True)
+ elif isinstance(chunk, dict):
+ model = chunk.get("model_id") or chunk.get("model")
+ if isinstance(model, str):
+ self.model = model
+ if isinstance(chunk.get("id"), str):
+ self.id = chunk["id"]
+ for position, choice in enumerate(chunk.get("choices") or []):
+ index = _number(_field(choice, "index")) or position
+ message = _field(choice, "delta") or _field(choice, "message") or {}
+ self._add(index, _text_content(_field(message, "content")))
+ if isinstance(_field(message, "role"), str):
+ self.role[index] = message["role"]
+ if isinstance(_field(choice, "finish_reason"), str):
+ self.finish[index] = choice["finish_reason"]
+ for position, result in enumerate(chunk.get("results") or []):
+ if not isinstance(result, dict):
+ continue
+ index = offset + position
+ self._add(index, result.get("generated_text"))
+ reason = result.get("stop_reason")
+ if isinstance(reason, str) and reason != "not_finished":
+ self.finish[index] = reason
+ self._tokens("tokens_in", result.get("input_token_count"), total)
+ self._tokens("tokens_out", result.get("generated_token_count"), total)
+ embedding = result.get("embedding")
+ if isinstance(embedding, (list, tuple)):
+ self.vectors += 1
+ self.dimension = len(embedding)
+ usage = chunk.get("usage")
+ if isinstance(usage, dict):
+ self._tokens("tokens_in", usage.get("prompt_tokens"), False)
+ self._tokens("tokens_out", usage.get("completion_tokens"), False)
+ self._tokens("tokens_total", usage.get("total_tokens"), False)
+ if "input_token_count" in chunk: # embeddings response
+ self._tokens("tokens_in", chunk.get("input_token_count"), total)
+
+ def apply(self, span: Any, operation: str, complete: bool = True) -> None:
+ if self.model:
+ span.set_attribute(A.GEN_AI_RESPONSE_MODEL, self.model)
+ if self.id:
+ span.set_attribute(A.GEN_AI_RESPONSE_ID, self.id)
+ if self.tokens_in is not None:
+ span.set_attribute(A.GEN_AI_USAGE_INPUT_TOKENS, self.tokens_in)
+ if self.tokens_out is not None:
+ span.set_attribute(A.GEN_AI_USAGE_OUTPUT_TOKENS, self.tokens_out)
+ total = self.tokens_total
+ if total is None and self.tokens_in is not None and self.tokens_out is not None:
+ total = self.tokens_in + self.tokens_out
+ if total is not None:
+ span.set_attribute(A.GEN_AI_USAGE_TOTAL_TOKENS, total)
+ if self.dimension is not None:
+ span.set_attribute(A.GEN_AI_EMBEDDINGS_DIMENSION_COUNT, self.dimension)
+ if self.finish:
+ span.set_attribute(A.GEN_AI_RESPONSE_FINISH_REASONS, [self.finish[i] for i in sorted(self.finish)])
+ if operation == "embeddings" or not complete:
+ return # vectors are never exported; abandoned streams export no partial output
+ for position, index in enumerate(sorted(self.text)):
+ prefix = f"{A.GEN_AI_OUTPUT_MESSAGES}.{position}.message"
+ span.set_attribute(f"{prefix}.role", self.role.get(index, "assistant"))
+ span.set_attribute(f"{prefix}.content", "".join(self.text[index]))
+
+
+def _start(tracer: Any, instance: Any, operation: str, args: tuple, kwargs: dict, method: str):
+ embeddings = operation == "embeddings"
+ model = getattr(instance, "model_id", None) or getattr(instance, "deployment_id", None)
+ attrs: dict[str, Any] = {A.GEN_AI_PROVIDER_NAME: PROVIDER, A.GEN_AI_OPERATION_NAME: operation}
+ if isinstance(model, str):
+ attrs[A.GEN_AI_REQUEST_MODEL] = model
+ params_index = 1
+ params = _params(_argument(args, kwargs, params_index, "params"))
+ if embeddings and method in ("embed_query", "aembed_query", "embed_documents", "aembed_documents"):
+ params = _params(_argument(args, kwargs, 1, "params"))
+ for attr, names, kind in _REQUEST_ATTRIBUTES:
+ for name in names:
+ number = _number(params.get(name), kind)
+ if number is not None:
+ attrs[attr] = number
+ break
+ stops = params.get("stop_sequences") or params.get("stop")
+ if isinstance(stops, (list, tuple)) and all(isinstance(s, str) for s in stops):
+ attrs[A.GEN_AI_REQUEST_STOP_SEQUENCES] = list(stops)
+ if params:
+ attrs[A.GEN_AI_REQUEST_PARAMETERS] = _json(params)
+
+ if embeddings:
+ inputs = _argument(args, kwargs, 0, "inputs") if method in ("generate", "agenerate") else (
+ _argument(args, kwargs, 0, "texts") if "documents" in method else _argument(args, kwargs, 0, "text"))
+ texts = [inputs] if isinstance(inputs, str) else inputs
+ for position, text in enumerate(texts if isinstance(texts, (list, tuple)) else []):
+ if isinstance(text, str):
+ attrs[f"{A.EMBEDDING_EMBEDDINGS}.{position}.embedding.text"] = text[:_LIMIT]
+ elif operation == "chat":
+ messages = _argument(args, kwargs, 0, "messages")
+ position = 0
+ for message in messages if isinstance(messages, (list, tuple)) else []:
+ content = _text_content(_field(message, "content"))
+ role = _field(message, "role")
+ if isinstance(role, str):
+ attrs[f"{A.GEN_AI_INPUT_MESSAGES}.{position}.message.role"] = role
+ if content is not None:
+ attrs[f"{A.GEN_AI_INPUT_MESSAGES}.{position}.message.content"] = content
+ position += 1
+ else:
+ prompt = _argument(args, kwargs, 0, "prompt")
+ prompts = [prompt] if isinstance(prompt, str) else prompt
+ for position, text in enumerate(prompts if isinstance(prompts, (list, tuple)) else []):
+ if isinstance(text, str):
+ attrs[f"{A.GEN_AI_INPUT_MESSAGES}.{position}.message.role"] = "user"
+ attrs[f"{A.GEN_AI_INPUT_MESSAGES}.{position}.message.content"] = text[:_LIMIT]
+ kind = FiSpanKindValues.EMBEDDING if embeddings else FiSpanKindValues.LLM
+ return tracer.start_span(f"watsonx.{operation}", attributes=attrs, fi_span_kind=kind)
+
+
+class _Stream:
+ """Passes chunks through unchanged; the span ends when the stream ends or is closed."""
+
+ def __init__(self, source: Any, span: Any, operation: str):
+ self._source, self._span, self._operation = source, span, operation
+ self._output = _Output()
+ self._ended = False
+
+ def __getattr__(self, name: str) -> Any:
+ return getattr(self._source, name)
+
+ def _finish(self, complete: bool = True) -> None:
+ if self._ended:
+ return
+ self._ended = True
+ try:
+ if not complete:
+ self._span.set_attribute("watsonx.stream.incomplete", True)
+ self._output.apply(self._span, self._operation, complete)
+ finally:
+ self._span.end()
+
+ def _fail(self, exc: BaseException) -> None:
+ if not self._ended:
+ self._ended = True
+ _failure(self._span, exc)
+
+ def _observe(self, chunk: Any) -> None:
+ try:
+ self._output.feed(chunk)
+ except Exception: # never let tracing break the caller's stream
+ pass
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ try:
+ with trace.use_span(self._span, end_on_exit=False, record_exception=False, set_status_on_exception=False):
+ chunk = next(self._source)
+ except StopIteration:
+ self._finish()
+ raise
+ except BaseException as exc:
+ self._fail(exc)
+ raise
+ self._observe(chunk)
+ return chunk
+
+ def __aiter__(self):
+ return self
+
+ async def __anext__(self):
+ try:
+ with trace.use_span(self._span, end_on_exit=False, record_exception=False, set_status_on_exception=False):
+ chunk = await self._source.__anext__()
+ except StopAsyncIteration:
+ self._finish()
+ raise
+ except BaseException as exc:
+ self._fail(exc)
+ raise
+ self._observe(chunk)
+ return chunk
+
+ def close(self):
+ try:
+ return self._source.close()
+ finally:
+ self._finish(complete=False)
+
+ async def aclose(self):
+ try:
+ return await self._source.aclose()
+ finally:
+ self._finish(complete=False)
+
+ def __del__(self):
+ try:
+ self._finish(complete=False)
+ except Exception:
+ pass
+
+
+def _complete(span: Any, result: Any, operation: str) -> Any:
+ if hasattr(result, "__anext__") or (hasattr(result, "__next__") and hasattr(result, "__iter__")):
+ return _Stream(result, span, operation)
+ output = _Output()
+ try:
+ output.feed(result, total=True)
+ output.apply(span, operation)
+ finally:
+ span.end()
+ return result
+
+
+def _wrap(tracer: Any, operation: str, method: str, is_async: bool):
+ def prepare(instance: Any, args: tuple):
+ if instance is None and args: # called as ``Class.method(self, ...)``
+ return args[0], args[1:]
+ return instance, args
+
+ if is_async:
+ async def wrapped(wrapped, instance, args, kwargs):
+ if _INSIDE.get():
+ return await wrapped(*args, **kwargs)
+ owner, rest = prepare(instance, args)
+ span = _start(tracer, owner, operation, rest, kwargs, method)
+ token = _INSIDE.set(True)
+ try:
+ with trace.use_span(span, end_on_exit=False, record_exception=False, set_status_on_exception=False):
+ result = await wrapped(*args, **kwargs)
+ except BaseException as exc:
+ _failure(span, exc)
+ raise
+ finally:
+ _INSIDE.reset(token)
+ return _complete(span, result, operation)
+ else:
+ def wrapped(wrapped, instance, args, kwargs):
+ if _INSIDE.get():
+ return wrapped(*args, **kwargs)
+ owner, rest = prepare(instance, args)
+ span = _start(tracer, owner, operation, rest, kwargs, method)
+ token = _INSIDE.set(True)
+ try:
+ with trace.use_span(span, end_on_exit=False, record_exception=False, set_status_on_exception=False):
+ result = wrapped(*args, **kwargs)
+ except BaseException as exc:
+ _failure(span, exc)
+ raise
+ finally:
+ _INSIDE.reset(token)
+ return _complete(span, result, operation)
+ return wrapped
+
+
+class WatsonxInstrumentor(BaseInstrumentor):
+ """Trace ``ModelInference`` text generation and chat, and ``Embeddings``, in ibm-watsonx-ai."""
+
+ def instrumentation_dependencies(self) -> Collection[str]:
+ return ("ibm-watsonx-ai >= 1.3.0, < 2",)
+
+ def _instrument(self, **kwargs: Any) -> None:
+ config = kwargs.get("config") or TraceConfig()
+ if not isinstance(config, TraceConfig):
+ raise TypeError("config must be a TraceConfig")
+ tracer = FITracer(trace.get_tracer(__name__, __version__, kwargs.get("tracer_provider")), config)
+ self._originals = []
+ for module, cls_name, method, operation, is_async in _METHODS:
+ cls = getattr(import_module(module), cls_name)
+ original = cls.__dict__.get(method)
+ if original is None: # this SDK version does not have the method
+ continue
+ wrap_function_wrapper(module, f"{cls_name}.{method}", _wrap(tracer, operation, method, is_async))
+ self._originals.append((cls, method, original))
+
+ def _uninstrument(self, **kwargs: Any) -> None:
+ for cls, name, original in getattr(self, "_originals", []):
+ setattr(cls, name, original)
+ self._originals = []
+
+
+__all__ = ["WatsonxInstrumentor", "__version__"]
diff --git a/python/frameworks/watsonx/traceai_watsonx/version.py b/python/frameworks/watsonx/traceai_watsonx/version.py
new file mode 100644
index 000000000..75716eb9a
--- /dev/null
+++ b/python/frameworks/watsonx/traceai_watsonx/version.py
@@ -0,0 +1 @@
+__version__ = "0.1.0" # x-release-please-version
diff --git a/python/tests/test_auto_entrypoints.py b/python/tests/test_auto_entrypoints.py
index 6a601780a..7be4a4b7d 100644
--- a/python/tests/test_auto_entrypoints.py
+++ b/python/tests/test_auto_entrypoints.py
@@ -348,6 +348,7 @@ def test_registry_is_consistent():
("transformers", "transformers", "traceAI-transformers", "traceai_transformers", "TransformersInstrumentor"),
("openrouter", "openrouter", "traceAI-openrouter", "traceai_openrouter", "OpenRouterInstrumentor"),
("marqo", "marqo", "traceAI-marqo", "traceai_marqo", "MarqoInstrumentor"),
+ ("watsonx", "ibm_watsonx_ai", "traceAI-watsonx", "traceai_watsonx", "WatsonxInstrumentor"),
])
def test_registry_enables_transformers_and_openrouter(name, target, package, module, entry):
integration = _registry.find(name)
diff --git a/release-please-config.json b/release-please-config.json
index 647c5065f..c400ac422 100644
--- a/release-please-config.json
+++ b/release-please-config.json
@@ -1346,6 +1346,14 @@
"traceai_marqo/version.py"
]
},
+ "python/frameworks/watsonx": {
+ "release-type": "python",
+ "component": "traceai-watsonx",
+ "package-name": "traceAI-watsonx",
+ "extra-files": [
+ "traceai_watsonx/version.py"
+ ]
+ },
"python/frameworks/openrouter": {
"release-type": "python",
"component": "traceai-openrouter",
From c85737a43c0cfa418bd769e0cef5706f81efa4d9 Mon Sep 17 00:00:00 2001
From: nik13 <8347993+nik13@users.noreply.github.com>
Date: Thu, 19 Nov 2026 03:34:20 +0000
Subject: [PATCH 3/8] test(watsonx): cover usage on delegating calls, instance
params, template variables, input/output values and tracing failures
---
.../watsonx/tests/test_instrumentation.py | 161 ++++++++++++++++++
1 file changed, 161 insertions(+)
diff --git a/python/frameworks/watsonx/tests/test_instrumentation.py b/python/frameworks/watsonx/tests/test_instrumentation.py
index 3cf54b23f..944cdcbe5 100644
--- a/python/frameworks/watsonx/tests/test_instrumentation.py
+++ b/python/frameworks/watsonx/tests/test_instrumentation.py
@@ -58,6 +58,11 @@ def __call__(self, req):
{"id": "c1", "model_id": LLM, "choices": [{"index": 0, "delta": {"content": "!"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 4, "completion_tokens": 2, "total_tokens": 6}},
))
+ if url.endswith("text/generation") and json.loads(req.content).get("input", "").startswith("echo:"):
+ text = json.loads(req.content)["input"] # per-prompt answers, to tell the prompts apart
+ return httpx.Response(200, json={"model_id": LLM, "results": [
+ {"generated_text": "out:" + text, "generated_token_count": len(text), "input_token_count": 4,
+ "stop_reason": "eos_token", "input_text": text}]})
if url.endswith("text/generation"):
return httpx.Response(200, json={"model_id": LLM, "results": [
{"generated_text": "Tokyo", "generated_token_count": 3, "input_token_count": 5, "stop_reason": "eos_token"}]})
@@ -350,3 +355,159 @@ def test_uninstrument_restores_originals_and_stops_tracing(api):
assert ModelInference.chat is before
ModelInference(model_id=LLM, api_client=api).chat(messages=[{"role": "user", "content": "hi"}])
assert exporter.get_finished_spans() == ()
+
+
+# ---- round 2 regressions -------------------------------------------------------------------------
+
+
+def test_generate_text_records_usage_finish_reason_and_response_model(spans, model):
+ assert model.generate_text(prompt="Capital of Japan?") == "Tokyo"
+ _, a = one(spans)
+ assert a["gen_ai.usage.input_tokens"] == 5
+ assert a["gen_ai.usage.output_tokens"] == 3
+ assert a["gen_ai.usage.total_tokens"] == 8
+ assert tuple(a["gen_ai.response.finish_reasons"]) == ("eos_token",)
+ assert a["gen_ai.response.model"] == LLM
+
+
+def test_embed_query_and_documents_record_usage_and_dimension(spans, api):
+ emb = Embeddings(model_id=EMB, api_client=api)
+ emb.embed_documents(["alpha", "beta"])
+ emb.embed_query("gamma")
+ finished = [dict(s.attributes) for s in spans.get_finished_spans()]
+ assert len(finished) == 2
+ for a in finished:
+ assert a["gen_ai.usage.input_tokens"] == 7
+ assert a["gen_ai.embeddings.dimension.count"] == 3
+ assert a["gen_ai.response.model"] == EMB
+
+
+def test_instance_level_params_are_recorded_and_call_params_win(spans, api):
+ model = ModelInference(model_id=LLM, api_client=api, params={"max_new_tokens": 11, "temperature": 0.3})
+ model.generate_text(prompt="x")
+ model.generate_text(prompt="x", params={"temperature": 0.9})
+ first, second = (dict(s.attributes) for s in spans.get_finished_spans())
+ assert first["gen_ai.request.max_tokens"] == 11 and first["gen_ai.request.temperature"] == 0.3
+ assert json.loads(first["gen_ai.request.parameters"]) == {"max_new_tokens": 11, "temperature": 0.3}
+ assert second["gen_ai.request.max_tokens"] == 11 and second["gen_ai.request.temperature"] == 0.9
+
+
+def test_instance_level_chat_and_embedding_params(spans, api):
+ ModelInference(model_id=LLM, api_client=api, params={"max_tokens": 5, "top_p": 0.4}).chat(
+ messages=[{"role": "user", "content": "hi"}])
+ Embeddings(model_id=EMB, api_client=api, params={"truncate_input_tokens": 3}).embed_query("q")
+ chat, emb = (dict(s.attributes) for s in spans.get_finished_spans())
+ assert chat["gen_ai.request.max_tokens"] == 5 and chat["gen_ai.request.top_p"] == 0.4
+ assert json.loads(emb["gen_ai.request.parameters"]) == {"truncate_input_tokens": 3}
+
+
+def test_prompt_variables_are_not_request_parameters(spans, model):
+ model.generate_text(prompt="x", params={"max_new_tokens": 4, "prompt_variables": {"name": "SECRETVAL"}})
+ _, a = one(spans)
+ assert "SECRETVAL" not in a["gen_ai.request.parameters"]
+ assert json.loads(a["gen_ai.request.parameters"]) == {"max_new_tokens": 4}
+ assert json.loads(a["gen_ai.prompt.template.variables"]) == {"name": "SECRETVAL"}
+
+
+@pytest.mark.parametrize("flag", ["hide_inputs", "hide_input_messages", "hide_input_text"])
+def test_prompt_variables_follow_the_input_hiding_flags(api, wire, flag):
+ exporter, provider, instrumentor = make_spans(TraceConfig(**{flag: True}))
+ try:
+ ModelInference(model_id=LLM, api_client=api).generate_text(
+ prompt="x", params={"max_new_tokens": 4, "prompt_variables": {"name": "SECRETVAL"}})
+ (span,) = exporter.get_finished_spans()
+ assert "SECRETVAL" not in json.dumps(dict(span.attributes), default=str)
+ finally:
+ instrumentor.uninstrument()
+ provider.shutdown()
+
+
+def test_input_and_output_value_are_set_like_sibling_providers(spans, model):
+ model.chat(messages=[{"role": "system", "content": "Be brief."}, {"role": "user", "content": "Capital?"}])
+ model.generate_text(prompt="Capital of Japan?")
+ list(model.generate_text_stream(prompt="Hi"))
+ chat, text, stream = (dict(s.attributes) for s in spans.get_finished_spans())
+ assert json.loads(chat["input.value"]) == [
+ {"role": "system", "content": "Be brief."}, {"role": "user", "content": "Capital?"}]
+ assert chat["input.mime_type"] == "application/json"
+ assert chat["output.value"] == "Tokyo"
+ assert json.loads(text["input.value"]) == "Capital of Japan?" and text["output.value"] == "Tokyo"
+ assert stream["output.value"] == "Hello"
+
+
+def test_input_output_value_never_include_images_or_embedding_vectors(spans, model, api):
+ model.chat(messages=[{"role": "user", "content": [
+ {"type": "text", "text": "what is this"}, {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}]}])
+ Embeddings(model_id=EMB, api_client=api).embed_query("q")
+ chat, emb = (dict(s.attributes) for s in spans.get_finished_spans())
+ assert "input.value" not in chat or "AAAA" not in chat["input.value"]
+ assert "AAAA" not in json.dumps(chat, default=str)
+ assert "input.value" not in emb and "output.value" not in emb
+
+
+@pytest.mark.parametrize("flags", [
+ {"hide_inputs": True, "hide_outputs": True},
+ {"hide_input_messages": True, "hide_output_messages": True},
+ {"hide_input_text": True, "hide_output_text": True},
+])
+def test_input_output_value_follow_the_hiding_flags(api, wire, flags):
+ exporter, provider, instrumentor = make_spans(TraceConfig(**flags))
+ try:
+ model = ModelInference(model_id=LLM, api_client=api)
+ model.chat(messages=[{"role": "user", "content": "my private question"}])
+ model.generate_text(prompt="my private prompt")
+ for span in exporter.get_finished_spans():
+ dumped = json.dumps(dict(span.attributes), default=str)
+ assert "private" not in dumped and "Tokyo" not in dumped
+ finally:
+ instrumentor.uninstrument()
+ provider.shutdown()
+
+
+def test_tracing_failure_in_start_never_breaks_the_call(model, monkeypatch):
+ import traceai_watsonx
+
+ exporter, provider, instrumentor = make_spans()
+ try:
+ def boom(*a, **k):
+ raise RuntimeError("tracing bug")
+
+ monkeypatch.setattr(traceai_watsonx, "_start", boom)
+ assert model.generate_text(prompt="x") == "Tokyo"
+ assert model.chat(messages=[{"role": "user", "content": "hi"}])["choices"][0]["message"]["content"] == "Tokyo"
+ assert exporter.get_finished_spans() == ()
+ finally:
+ instrumentor.uninstrument()
+ provider.shutdown()
+
+
+def test_tracing_failure_in_result_handling_never_breaks_the_call(model, monkeypatch):
+ import traceai_watsonx
+
+ exporter, provider, instrumentor = make_spans()
+ try:
+ def boom(*a, **k):
+ raise RuntimeError("tracing bug")
+
+ monkeypatch.setattr(traceai_watsonx._Output, "feed", boom)
+ monkeypatch.setattr(traceai_watsonx._Output, "apply", boom)
+ assert model.generate_text(prompt="x") == "Tokyo"
+ assert model.chat(messages=[{"role": "user", "content": "hi"}])["choices"][0]["message"]["content"] == "Tokyo"
+ assert len(exporter.get_finished_spans()) == 2 # spans still end
+ finally:
+ instrumentor.uninstrument()
+ provider.shutdown()
+
+
+@pytest.mark.parametrize("async_mode", [False, True])
+def test_prompt_list_outputs_are_indexed_per_prompt_and_usage_is_summed(spans, model, async_mode):
+ prompts = ["echo:aaa", "echo:bbbbb"]
+ result = model.generate(prompt=prompts, async_mode=async_mode)
+ list(result) if async_mode else result
+ _, a = one(spans)
+ assert a["gen_ai.output.messages.0.message.content"] == "out:echo:aaa"
+ assert a["gen_ai.output.messages.1.message.content"] == "out:echo:bbbbb"
+ assert a["gen_ai.input.messages.1.message.content"] == "echo:bbbbb"
+ assert a["gen_ai.usage.input_tokens"] == 8
+ assert a["gen_ai.usage.output_tokens"] == len("echo:aaa") + len("echo:bbbbb")
+ assert tuple(a["gen_ai.response.finish_reasons"]) == ("eos_token", "eos_token")
From 811c76aaa8d16d601b89f7ed72cd442baeceb7a9 Mon Sep 17 00:00:00 2001
From: nik13 <8347993+nik13@users.noreply.github.com>
Date: Thu, 19 Nov 2026 03:36:19 +0000
Subject: [PATCH 4/8] fix(watsonx): record usage on delegating calls and
instance params, keep template variables out of request parameters, set
input/output values, guard tracing failures
- generate_text, embed_query and embed_documents take usage, finish reason and response model from the inner generate response through the outer span's accumulator.
- Parameters given when creating ModelInference or Embeddings are recorded under the call's.
- prompt_variables move to gen_ai.prompt.template.variables, which core hides with the input flags.
- input.value, input.mime_type and output.value follow the sibling providers and the hide settings.
- A tracing error in span start or result handling never breaks a successful call.
- generate(prompt=[...], async_mode=True) records each answer under its own prompt and sums usage.
---
python/frameworks/watsonx/README.md | 2 +-
.../watsonx/traceai_watsonx/__init__.py | 171 ++++++++++++++----
2 files changed, 133 insertions(+), 40 deletions(-)
diff --git a/python/frameworks/watsonx/README.md b/python/frameworks/watsonx/README.md
index 9500f7b90..98144003b 100644
--- a/python/frameworks/watsonx/README.md
+++ b/python/frameworks/watsonx/README.md
@@ -39,7 +39,7 @@ Find the `watsonx.chat` span in your Future AGI project. Covered calls:
| `watsonx.chat` | `ModelInference.chat`, `chat_stream`, `achat`, `achat_stream` |
| `watsonx.embeddings` | `Embeddings.generate`, `embed_documents`, `embed_query`, `agenerate`, `aembed_documents`, `aembed_query` |
-Each span records `gen_ai.provider.name` (`ibm.watsonx.ai`), the request model or deployment id, request parameters, input and output messages, token usage (from `input_token_count` / `generated_token_count` or the chat `usage`), and finish reasons. `generate_text` returns only the text, so its span has no token counts; use `generate` or `generate_text(raw_response=True)` to get them. One SDK call is one span: methods that call each other inside the SDK (`generate_text` calls `generate`, `embed_query` calls `generate`) are not traced twice.
+Each span records `gen_ai.provider.name` (`ibm.watsonx.ai`), the request model or deployment id, request parameters (those given to the call, over those the `ModelInference` or `Embeddings` object was created with), input and output messages with `input.value` / `output.value`, token usage (from `input_token_count` / `generated_token_count`, the embeddings `input_token_count`, or the chat `usage`), and finish reasons. One SDK call is one span: methods that call each other inside the SDK (`generate_text` calls `generate`, `embed_query` calls `generate`) are not traced twice, and the outer span takes its usage, finish reason and response model from the inner response. Only a plain-text `generate_text_stream` has no token counts, because it yields text only; use `raw_response=True` to get them. Prompt-template values (`prompt_variables`) are recorded as `gen_ai.prompt.template.variables`, not as a request parameter, and are removed by the input-hiding settings. For `generate(prompt=[...])` each answer is recorded under its own prompt's message index and usage is summed.
Streams finish when fully consumed or closed; a stream abandoned without closing it is marked `watsonx.stream.incomplete` and carries no partial output. Errors set the span status to ERROR and `error.type`, and the exception is re-raised unchanged. Exception messages, stacks, credentials and embedding vectors are never exported. Non-text parts of chat messages (images) are not captured.
diff --git a/python/frameworks/watsonx/traceai_watsonx/__init__.py b/python/frameworks/watsonx/traceai_watsonx/__init__.py
index 6702a1c97..8cb323977 100644
--- a/python/frameworks/watsonx/traceai_watsonx/__init__.py
+++ b/python/frameworks/watsonx/traceai_watsonx/__init__.py
@@ -44,9 +44,11 @@
(_EMBEDDINGS, "Embeddings", "aembed_query", "embeddings", True),
)
-# True while a traced SDK call is running, so the SDK's internal calls to other public methods
-# do not create nested duplicate spans.
-_INSIDE: contextvars.ContextVar[bool] = contextvars.ContextVar("traceai_watsonx_inside", default=False)
+# The accumulator of the traced SDK call that is running, or None. The SDK's internal calls to
+# other public methods do not create nested duplicate spans; their results feed this accumulator,
+# so ``generate_text`` and ``embed_query`` still record usage and finish reason from ``generate``.
+_INSIDE: contextvars.ContextVar[Optional["_Output"]] = contextvars.ContextVar(
+ "traceai_watsonx_inside", default=None)
_REQUEST_ATTRIBUTES = (
(A.GEN_AI_REQUEST_MAX_TOKENS, ("max_new_tokens", "max_tokens", "max_completion_tokens"), int),
@@ -104,6 +106,18 @@ def _text_content(content: Any) -> Optional[str]:
return None
+def _is_stream(result: Any) -> bool:
+ return hasattr(result, "__anext__") or (hasattr(result, "__next__") and hasattr(result, "__iter__"))
+
+
+def _capture_input(config: TraceConfig) -> bool:
+ return not (config.hide_inputs or config.hide_input_text or config.hide_input_messages or config.hide_input_images)
+
+
+def _capture_output(config: TraceConfig) -> bool:
+ return not (config.hide_outputs or config.hide_output_text or config.hide_output_messages)
+
+
def _failure(span: Any, exc: BaseException) -> None:
# Exception text may carry the request body, URL or credentials: record only the type.
span.set_attribute("error.type", type(exc).__name__)
@@ -126,6 +140,7 @@ def __init__(self) -> None:
self.vectors = 0
self.dimension: Optional[int] = None
self.size = 0
+ self.absorbed = False # an inner SDK call already supplied the response
def _add(self, index: int, piece: Any) -> None:
if isinstance(piece, str) and piece and self.size < _LIMIT:
@@ -189,7 +204,17 @@ def feed(self, chunk: Any, offset: int = 0, total: bool = False) -> None:
if "input_token_count" in chunk: # embeddings response
self._tokens("tokens_in", chunk.get("input_token_count"), total)
- def apply(self, span: Any, operation: str, complete: bool = True) -> None:
+ def absorb(self, result: Any) -> None:
+ """Take the result of an inner SDK call made by the traced call (never a stream)."""
+ if _is_stream(result):
+ return
+ try:
+ self.feed(result, total=True)
+ self.absorbed = True
+ except Exception:
+ pass
+
+ def apply(self, span: Any, operation: str, complete: bool = True, config: Optional[TraceConfig] = None) -> None:
if self.model:
span.set_attribute(A.GEN_AI_RESPONSE_MODEL, self.model)
if self.id:
@@ -213,18 +238,35 @@ def apply(self, span: Any, operation: str, complete: bool = True) -> None:
prefix = f"{A.GEN_AI_OUTPUT_MESSAGES}.{position}.message"
span.set_attribute(f"{prefix}.role", self.role.get(index, "assistant"))
span.set_attribute(f"{prefix}.content", "".join(self.text[index]))
+ if config is not None and _capture_output(config):
+ texts = ["".join(self.text[index]) for index in sorted(self.text)]
+ if len(texts) == 1:
+ span.set_attribute(A.OUTPUT_VALUE, texts[0])
+ elif texts:
+ span.set_attribute(A.OUTPUT_VALUE, json.dumps(texts)[:_LIMIT])
+ span.set_attribute(A.OUTPUT_MIME_TYPE, "application/json")
+
+
+def _prompts(operation: str, args: tuple, kwargs: dict) -> list:
+ if operation != "text_completion":
+ return []
+ prompt = _argument(args, kwargs, 0, "prompt")
+ return [prompt] if isinstance(prompt, str) else (list(prompt) if isinstance(prompt, (list, tuple)) else [])
-def _start(tracer: Any, instance: Any, operation: str, args: tuple, kwargs: dict, method: str):
+def _start(tracer: Any, config: TraceConfig, instance: Any, operation: str, args: tuple, kwargs: dict, method: str):
embeddings = operation == "embeddings"
model = getattr(instance, "model_id", None) or getattr(instance, "deployment_id", None)
attrs: dict[str, Any] = {A.GEN_AI_PROVIDER_NAME: PROVIDER, A.GEN_AI_OPERATION_NAME: operation}
if isinstance(model, str):
attrs[A.GEN_AI_REQUEST_MODEL] = model
- params_index = 1
- params = _params(_argument(args, kwargs, params_index, "params"))
- if embeddings and method in ("embed_query", "aembed_query", "embed_documents", "aembed_documents"):
- params = _params(_argument(args, kwargs, 1, "params"))
+ # Parameters set when the SDK object was created apply to every call; call-level ones win.
+ params = {**_params(getattr(instance, "params", None)), **_params(_argument(args, kwargs, 1, "params"))}
+ # Template input is user content: it goes under the key core hides with the input flags,
+ # never into the request parameters (core hides those only under hide_inputs).
+ variables = params.pop("prompt_variables", None)
+ if isinstance(variables, dict) and variables:
+ attrs[A.GEN_AI_PROMPT_TEMPLATE_VARIABLES] = _json(variables)
for attr, names, kind in _REQUEST_ATTRIBUTES:
for name in names:
number = _number(params.get(name), kind)
@@ -247,6 +289,7 @@ def _start(tracer: Any, instance: Any, operation: str, args: tuple, kwargs: dict
elif operation == "chat":
messages = _argument(args, kwargs, 0, "messages")
position = 0
+ plain = []
for message in messages if isinstance(messages, (list, tuple)) else []:
content = _text_content(_field(message, "content"))
role = _field(message, "role")
@@ -254,14 +297,23 @@ def _start(tracer: Any, instance: Any, operation: str, args: tuple, kwargs: dict
attrs[f"{A.GEN_AI_INPUT_MESSAGES}.{position}.message.role"] = role
if content is not None:
attrs[f"{A.GEN_AI_INPUT_MESSAGES}.{position}.message.content"] = content
+ if isinstance(_field(message, "content"), str):
+ plain.append({"role": role, "content": content})
position += 1
+ # Only plain-text conversations become input.value: it cannot tell text from an inline image.
+ if messages and len(plain) == position == len(messages) and _capture_input(config):
+ attrs[A.INPUT_VALUE] = json.dumps(plain)[:_LIMIT]
+ attrs[A.INPUT_MIME_TYPE] = "application/json"
else:
- prompt = _argument(args, kwargs, 0, "prompt")
- prompts = [prompt] if isinstance(prompt, str) else prompt
- for position, text in enumerate(prompts if isinstance(prompts, (list, tuple)) else []):
+ prompts = _prompts(operation, args, kwargs)
+ for position, text in enumerate(prompts):
if isinstance(text, str):
attrs[f"{A.GEN_AI_INPUT_MESSAGES}.{position}.message.role"] = "user"
attrs[f"{A.GEN_AI_INPUT_MESSAGES}.{position}.message.content"] = text[:_LIMIT]
+ if prompts and all(isinstance(t, str) for t in prompts) and _capture_input(config):
+ value = prompts[0] if len(prompts) == 1 and isinstance(_argument(args, kwargs, 0, "prompt"), str) else prompts
+ attrs[A.INPUT_VALUE] = json.dumps(value)[:_LIMIT]
+ attrs[A.INPUT_MIME_TYPE] = "application/json"
kind = FiSpanKindValues.EMBEDDING if embeddings else FiSpanKindValues.LLM
return tracer.start_span(f"watsonx.{operation}", attributes=attrs, fi_span_kind=kind)
@@ -269,10 +321,14 @@ def _start(tracer: Any, instance: Any, operation: str, args: tuple, kwargs: dict
class _Stream:
"""Passes chunks through unchanged; the span ends when the stream ends or is closed."""
- def __init__(self, source: Any, span: Any, operation: str):
- self._source, self._span, self._operation = source, span, operation
+ def __init__(self, source: Any, span: Any, operation: str, config: TraceConfig, batch: Optional[list] = None):
+ self._source, self._span, self._operation, self._config = source, span, operation, config
self._output = _Output()
self._ended = False
+ # ``generate(prompt=[...], async_mode=True)`` yields one whole response per prompt, in
+ # completion order; ``batch`` holds the prompts so each answer lands on its own index.
+ self._batch = batch
+ self._seen: set[int] = set()
def __getattr__(self, name: str) -> Any:
return getattr(self._source, name)
@@ -284,7 +340,7 @@ def _finish(self, complete: bool = True) -> None:
try:
if not complete:
self._span.set_attribute("watsonx.stream.incomplete", True)
- self._output.apply(self._span, self._operation, complete)
+ self._output.apply(self._span, self._operation, complete, self._config)
finally:
self._span.end()
@@ -293,9 +349,23 @@ def _fail(self, exc: BaseException) -> None:
self._ended = True
_failure(self._span, exc)
+ def _slot(self, chunk: Any) -> int:
+ results = chunk.get("results") if isinstance(chunk, dict) else None
+ echoed = results[0].get("input_text") if results and isinstance(results[0], dict) else None
+ for index, prompt in enumerate(self._batch or []):
+ if index not in self._seen and prompt == echoed:
+ break
+ else:
+ index = next((i for i in range(len(self._batch or [])) if i not in self._seen), len(self._seen))
+ self._seen.add(index)
+ return index
+
def _observe(self, chunk: Any) -> None:
try:
- self._output.feed(chunk)
+ if self._batch is not None:
+ self._output.feed(chunk, offset=self._slot(chunk), total=True)
+ else:
+ self._output.feed(chunk)
except Exception: # never let tracing break the caller's stream
pass
@@ -350,31 +420,48 @@ def __del__(self):
pass
-def _complete(span: Any, result: Any, operation: str) -> Any:
- if hasattr(result, "__anext__") or (hasattr(result, "__next__") and hasattr(result, "__iter__")):
- return _Stream(result, span, operation)
- output = _Output()
+def _complete(span: Any, result: Any, operation: str, config: TraceConfig, output: _Output,
+ method: str = "", prompts: Optional[list] = None) -> Any:
+ if _is_stream(result):
+ # A generator from ``generate`` is the multi-prompt async_mode batch, not a token stream.
+ batch = prompts if method == "generate" and hasattr(result, "__next__") else None
+ return _Stream(result, span, operation, config, batch)
try:
- output.feed(result, total=True)
- output.apply(span, operation)
+ if not output.absorbed: # otherwise an inner SDK call already supplied the response
+ output.feed(result, total=True)
+ output.apply(span, operation, config=config)
+ except Exception: # never let tracing break a successful call
+ pass
finally:
- span.end()
+ try:
+ span.end()
+ except Exception:
+ pass
return result
-def _wrap(tracer: Any, operation: str, method: str, is_async: bool):
- def prepare(instance: Any, args: tuple):
- if instance is None and args: # called as ``Class.method(self, ...)``
- return args[0], args[1:]
- return instance, args
+def _wrap(tracer: Any, config: TraceConfig, operation: str, method: str, is_async: bool):
+ def begin(instance: Any, args: tuple, kwargs: dict):
+ """The span and its accumulator, or None when tracing itself fails."""
+ try:
+ owner, rest = (args[0], args[1:]) if instance is None and args else (instance, args)
+ return _start(tracer, config, owner, operation, rest, kwargs, method), _prompts(operation, rest, kwargs)
+ except Exception:
+ return None
if is_async:
async def wrapped(wrapped, instance, args, kwargs):
- if _INSIDE.get():
+ outer = _INSIDE.get()
+ if outer is not None:
+ result = await wrapped(*args, **kwargs)
+ outer.absorb(result)
+ return result
+ started = begin(instance, args, kwargs)
+ if started is None:
return await wrapped(*args, **kwargs)
- owner, rest = prepare(instance, args)
- span = _start(tracer, owner, operation, rest, kwargs, method)
- token = _INSIDE.set(True)
+ span, prompts = started
+ output = _Output()
+ token = _INSIDE.set(output)
try:
with trace.use_span(span, end_on_exit=False, record_exception=False, set_status_on_exception=False):
result = await wrapped(*args, **kwargs)
@@ -383,14 +470,20 @@ async def wrapped(wrapped, instance, args, kwargs):
raise
finally:
_INSIDE.reset(token)
- return _complete(span, result, operation)
+ return _complete(span, result, operation, config, output, method, prompts)
else:
def wrapped(wrapped, instance, args, kwargs):
- if _INSIDE.get():
+ outer = _INSIDE.get()
+ if outer is not None:
+ result = wrapped(*args, **kwargs)
+ outer.absorb(result)
+ return result
+ started = begin(instance, args, kwargs)
+ if started is None:
return wrapped(*args, **kwargs)
- owner, rest = prepare(instance, args)
- span = _start(tracer, owner, operation, rest, kwargs, method)
- token = _INSIDE.set(True)
+ span, prompts = started
+ output = _Output()
+ token = _INSIDE.set(output)
try:
with trace.use_span(span, end_on_exit=False, record_exception=False, set_status_on_exception=False):
result = wrapped(*args, **kwargs)
@@ -399,7 +492,7 @@ def wrapped(wrapped, instance, args, kwargs):
raise
finally:
_INSIDE.reset(token)
- return _complete(span, result, operation)
+ return _complete(span, result, operation, config, output, method, prompts)
return wrapped
@@ -420,7 +513,7 @@ def _instrument(self, **kwargs: Any) -> None:
original = cls.__dict__.get(method)
if original is None: # this SDK version does not have the method
continue
- wrap_function_wrapper(module, f"{cls_name}.{method}", _wrap(tracer, operation, method, is_async))
+ wrap_function_wrapper(module, f"{cls_name}.{method}", _wrap(tracer, config, operation, method, is_async))
self._originals.append((cls, method, original))
def _uninstrument(self, **kwargs: Any) -> None:
From 65e71965755ebefe13b40ce1ad5370556d543f92 Mon Sep 17 00:00:00 2001
From: nik13 <8347993+nik13@users.noreply.github.com>
Date: Thu, 19 Nov 2026 03:24:25 +0000
Subject: [PATCH 5/8] test(writer): cover Writer SDK chat, completions,
tool-calling and streaming tracing
---
python/frameworks/writer/tests/conftest.py | 111 ++++++
python/frameworks/writer/tests/test_writer.py | 376 ++++++++++++++++++
python/tests/test_auto_entrypoints.py | 10 +
3 files changed, 497 insertions(+)
create mode 100644 python/frameworks/writer/tests/conftest.py
create mode 100644 python/frameworks/writer/tests/test_writer.py
diff --git a/python/frameworks/writer/tests/conftest.py b/python/frameworks/writer/tests/conftest.py
new file mode 100644
index 000000000..151a4b166
--- /dev/null
+++ b/python/frameworks/writer/tests/conftest.py
@@ -0,0 +1,111 @@
+"""Offline Writer SDK harness: the real SDK client over an httpx MockTransport, no network."""
+import json
+
+import httpx
+import pytest
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+from writerai import AsyncWriter, Writer
+
+API_KEY = "synthetic-writer-key-DO-NOT-EXPORT"
+
+
+def make_client(handler):
+ transport = httpx.MockTransport(handler)
+ return Writer(
+ api_key=API_KEY,
+ base_url="http://writer.test",
+ max_retries=0,
+ http_client=httpx.Client(transport=transport),
+ )
+
+
+def make_async_client(handler):
+ transport = httpx.MockTransport(handler)
+ return AsyncWriter(
+ api_key=API_KEY,
+ base_url="http://writer.test",
+ max_retries=0,
+ http_client=httpx.AsyncClient(transport=transport),
+ )
+
+
+def chat_body(text="Tokyo", **message):
+ msg = {"role": "assistant", "content": text}
+ msg.update(message)
+ return {
+ "id": "chat-1",
+ "object": "chat.completion",
+ "created": 1,
+ "model": "palmyra-x5-served",
+ "choices": [{"index": 0, "finish_reason": "stop", "message": msg}],
+ "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5},
+ }
+
+
+def completion_body(text="done"):
+ return {"id": "cmpl-1", "choices": [{"text": text}], "model": "palmyra-x-003-instruct"}
+
+
+def sse(*events):
+ out = "".join(f"data: {json.dumps(e)}\n\n" for e in events) + "data: [DONE]\n\n"
+ return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=out.encode())
+
+
+def chunk(content=None, finish=None, usage=None, tool_calls=None, model="palmyra-x5-served"):
+ delta = {}
+ if content is not None:
+ delta["content"] = content
+ if tool_calls is not None:
+ delta["tool_calls"] = tool_calls
+ body = {
+ "id": "chat-1",
+ "object": "chat.completion.chunk",
+ "created": 1,
+ "model": model,
+ "choices": [{"index": 0, "delta": delta, "finish_reason": finish}],
+ }
+ if usage is not None:
+ body["usage"] = usage
+ return body
+
+
+def attrs(span):
+ return dict(span.attributes)
+
+
+def blob(span):
+ return json.dumps(attrs(span), default=str)
+
+
+@pytest.fixture
+def provider():
+ exporter = InMemorySpanExporter()
+ tracer_provider = TracerProvider()
+ tracer_provider.add_span_processor(SimpleSpanProcessor(exporter))
+ tracer_provider.exporter = exporter
+ yield tracer_provider
+ tracer_provider.shutdown()
+
+
+@pytest.fixture
+def instrument(provider):
+ from traceai_writer import WriterInstrumentor
+
+ instrumentors = []
+
+ def start(**kwargs):
+ instrumentor = WriterInstrumentor()
+ instrumentor.instrument(tracer_provider=provider, **kwargs)
+ instrumentors.append(instrumentor)
+ return provider.exporter
+
+ yield start
+ for instrumentor in reversed(instrumentors):
+ instrumentor.uninstrument()
+
+
+@pytest.fixture
+def spans(instrument):
+ return instrument()
diff --git a/python/frameworks/writer/tests/test_writer.py b/python/frameworks/writer/tests/test_writer.py
new file mode 100644
index 000000000..b18d3d2c6
--- /dev/null
+++ b/python/frameworks/writer/tests/test_writer.py
@@ -0,0 +1,376 @@
+"""Writer SDK instrumentation over the real SDK client with a stubbed HTTP transport."""
+import asyncio
+import json
+
+import httpx
+import pytest
+import writerai
+from opentelemetry import context as context_api
+from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY
+from opentelemetry.trace import StatusCode
+
+from conftest import (
+ API_KEY, attrs, blob, chat_body, chunk, completion_body, make_async_client, make_client, sse,
+)
+from fi_instrumentation import TraceConfig
+
+MESSAGES = [{"role": "user", "content": "Capital of Japan?"}]
+OUT = "gen_ai.output.messages.0.message"
+
+
+def one(spans):
+ finished = spans.get_finished_spans()
+ assert len(finished) == 1, [s.name for s in finished]
+ return finished[0]
+
+
+def ok_handler(body):
+ seen = []
+
+ def handler(request):
+ seen.append(request)
+ return httpx.Response(200, json=body)
+
+ handler.seen = seen
+ return handler
+
+
+# --- chat --------------------------------------------------------------------------------------
+
+def test_chat_records_model_parameters_usage_and_content(spans):
+ handler = ok_handler(chat_body())
+ client = make_client(handler)
+ result = client.chat.chat(
+ messages=MESSAGES, model="palmyra-x5", temperature=0.2, max_tokens=16, top_p=0.9, stop=["END"]
+ )
+ assert result.choices[0].message.content == "Tokyo"
+ # The SDK request is untouched.
+ sent = json.loads(handler.seen[0].content)
+ assert sent["messages"] == MESSAGES and sent["model"] == "palmyra-x5"
+ span = one(spans)
+ a = attrs(span)
+ assert span.name == "writer.chat"
+ assert a["gen_ai.span.kind"] == "LLM"
+ assert a["gen_ai.provider.name"] == "writer"
+ assert a["gen_ai.operation.name"] == "chat"
+ assert a["gen_ai.request.model"] == "palmyra-x5"
+ assert a["gen_ai.request.temperature"] == 0.2
+ assert a["gen_ai.request.max_tokens"] == 16
+ assert a["gen_ai.request.top_p"] == 0.9
+ assert list(a["gen_ai.request.stop_sequences"]) == ["END"]
+ assert a["gen_ai.response.model"] == "palmyra-x5-served"
+ assert a["gen_ai.response.id"] == "chat-1"
+ assert list(a["gen_ai.response.finish_reasons"]) == ["stop"]
+ assert (a["gen_ai.usage.input_tokens"], a["gen_ai.usage.output_tokens"], a["gen_ai.usage.total_tokens"]) == (3, 2, 5)
+ assert json.loads(a["input.value"]) == MESSAGES
+ assert a["output.value"] == "Tokyo"
+ assert API_KEY not in blob(span)
+ assert span.status.status_code != StatusCode.ERROR
+
+
+def test_chat_async(spans):
+ client = make_async_client(ok_handler(chat_body()))
+
+ async def go():
+ return await client.chat.chat(messages=MESSAGES, model="palmyra-x5")
+
+ assert asyncio.run(go()).choices[0].message.content == "Tokyo"
+ a = attrs(one(spans))
+ assert a["output.value"] == "Tokyo" and a["gen_ai.usage.total_tokens"] == 5
+
+
+def test_tool_calls_in_response_are_recorded(spans):
+ body = chat_body(
+ "",
+ tool_calls=[{"id": "call-1", "type": "function", "function": {"name": "get_weather", "arguments": '{"city": "Tokyo"}'}}],
+ )
+ body["choices"][0]["finish_reason"] = "tool_calls"
+ client = make_client(ok_handler(body))
+ tools = [{"type": "function", "function": {"name": "get_weather", "description": "d", "parameters": {"type": "object", "properties": {}}}}]
+ client.chat.chat(messages=MESSAGES, model="palmyra-x5", tools=tools)
+ a = attrs(one(spans))
+ prefix = f"{OUT}.tool_calls.0.tool_call"
+ assert a[f"{prefix}.id"] == "call-1"
+ assert a[f"{prefix}.function.name"] == "get_weather"
+ assert json.loads(a[f"{prefix}.function.arguments"]) == {"city": "Tokyo"}
+ assert list(a["gen_ai.response.finish_reasons"]) == ["tool_calls"]
+
+
+def test_tool_arguments_are_hidden_with_outputs(instrument):
+ spans = instrument(config=TraceConfig(hide_outputs=True))
+ body = chat_body("", tool_calls=[{"id": "c", "type": "function", "function": {"name": "lookup", "arguments": '{"ssn": "SECRET-ARG"}'}}])
+ make_client(ok_handler(body)).chat.chat(messages=MESSAGES, model="m")
+ span = one(spans)
+ assert "SECRET-ARG" not in blob(span)
+ assert attrs(span)["gen_ai.usage.total_tokens"] == 5
+
+
+def test_tool_result_messages_are_part_of_the_input(spans):
+ messages = [
+ {"role": "user", "content": "weather?"},
+ {"role": "assistant", "content": "", "tool_calls": [{"id": "c1", "type": "function", "function": {"name": "w", "arguments": "{}"}}]},
+ {"role": "tool", "tool_call_id": "c1", "content": "sunny"},
+ ]
+ make_client(ok_handler(chat_body("It is sunny"))).chat.chat(messages=messages, model="m")
+ a = attrs(one(spans))
+ parsed = json.loads(a["input.value"])
+ assert [m["role"] for m in parsed] == ["user", "assistant", "tool"]
+ assert parsed[2]["content"] == "sunny"
+
+
+def test_graph_documents_and_web_sources_are_not_exported(spans):
+ body = chat_body(
+ "answer",
+ graph_data={"sources": [{"file_id": "f1", "score": 0.9, "text": "CONFIDENTIAL-DOCUMENT-BODY"}], "status": "finished"},
+ web_search_data={"sources": [{"url": "http://x", "raw_content": "CONFIDENTIAL-WEB-BODY"}]},
+ )
+ make_client(ok_handler(body)).chat.chat(messages=MESSAGES, model="m", tools=[{"type": "graph", "function": {"graph_ids": ["g1"], "subqueries": True}}])
+ span = one(spans)
+ assert "CONFIDENTIAL" not in blob(span)
+ assert attrs(span)["output.value"] == "answer"
+
+
+def test_multimodal_or_nonstring_message_content_is_not_serialized(spans):
+ messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,SECRETPIXELS"}}]}]
+ make_client(ok_handler(chat_body())).chat.chat(messages=messages, model="m")
+ span = one(spans)
+ assert "SECRETPIXELS" not in blob(span)
+ assert "input.value" not in attrs(span)
+
+
+# --- streaming ---------------------------------------------------------------------------------
+
+def stream_handler():
+ return lambda request: sse(
+ chunk("Tok"),
+ chunk("yo"),
+ chunk(None, finish="stop", usage={"prompt_tokens": 6, "completion_tokens": 2, "total_tokens": 8}),
+ )
+
+
+def test_stream_ends_on_exhaustion_with_assembled_output(spans):
+ client = make_client(stream_handler())
+ stream = client.chat.chat(messages=MESSAGES, model="palmyra-x5", stream=True, stream_options={"include_usage": True})
+ assert spans.get_finished_spans() == ()
+ pieces = [c.choices[0].delta.content for c in stream]
+ assert pieces[:2] == ["Tok", "yo"]
+ span = one(spans)
+ a = attrs(span)
+ assert a["output.value"] == "Tokyo"
+ assert (a["gen_ai.usage.input_tokens"], a["gen_ai.usage.output_tokens"], a["gen_ai.usage.total_tokens"]) == (6, 2, 8)
+ assert list(a["gen_ai.response.finish_reasons"]) == ["stop"]
+ assert a["gen_ai.response.model"] == "palmyra-x5-served"
+
+
+def test_stream_async(spans):
+ client = make_async_client(stream_handler())
+
+ async def go():
+ stream = await client.chat.chat(messages=MESSAGES, model="m", stream=True)
+ return [c.choices[0].delta.content async for c in stream]
+
+ assert asyncio.run(go())[:2] == ["Tok", "yo"]
+ assert attrs(one(spans))["output.value"] == "Tokyo"
+
+
+def test_stream_close_ends_span_incomplete_once(spans):
+ stream = make_client(stream_handler()).chat.chat(messages=MESSAGES, model="m", stream=True)
+ next(iter(stream))
+ assert spans.get_finished_spans() == ()
+ stream.close()
+ span = one(spans)
+ assert attrs(span)["writer.stream.incomplete"] is True
+ stream.close()
+ assert len(spans.get_finished_spans()) == 1
+
+
+def test_stream_context_manager(spans):
+ with make_client(stream_handler()).chat.chat(messages=MESSAGES, model="m", stream=True) as stream:
+ for _ in stream:
+ pass
+ assert len(spans.get_finished_spans()) == 1
+
+
+def test_streamed_tool_call_deltas_are_assembled(spans):
+ handler = lambda request: sse(
+ chunk(None, tool_calls=[{"index": 0, "id": "call-9", "type": "function", "function": {"name": "get_weather", "arguments": '{"ci'}}]),
+ chunk(None, tool_calls=[{"index": 0, "function": {"arguments": 'ty": "Oslo"}'}}]),
+ chunk(None, finish="tool_calls"),
+ )
+ for _ in make_client(handler).chat.chat(messages=MESSAGES, model="m", stream=True):
+ pass
+ a = attrs(one(spans))
+ prefix = f"{OUT}.tool_calls.0.tool_call"
+ assert a[f"{prefix}.id"] == "call-9"
+ assert a[f"{prefix}.function.name"] == "get_weather"
+ assert json.loads(a[f"{prefix}.function.arguments"]) == {"city": "Oslo"}
+
+
+def test_chat_stream_helper_produces_exactly_one_span(spans):
+ with make_client(stream_handler()).chat.stream(messages=MESSAGES, model="palmyra-x5") as stream:
+ for _ in stream:
+ pass
+ span = one(spans)
+ assert span.name == "writer.chat"
+ assert attrs(span)["output.value"] == "Tokyo"
+
+
+def test_stream_hidden_outputs(instrument):
+ spans = instrument(config=TraceConfig(hide_outputs=True))
+ for _ in make_client(stream_handler()).chat.chat(messages=MESSAGES, model="m", stream=True):
+ pass
+ span = one(spans)
+ assert "Tokyo" not in blob(span)
+ assert attrs(span)["gen_ai.usage.total_tokens"] == 8
+
+
+def test_stream_error_midway(spans):
+ class Boom(httpx.SyncByteStream):
+ def __iter__(self):
+ yield b'data: ' + json.dumps(chunk("a")).encode() + b"\n\n"
+ raise httpx.ReadError("connection lost SECRETDETAIL")
+
+ handler = lambda request: httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=Boom())
+ stream = make_client(handler).chat.chat(messages=MESSAGES, model="m", stream=True)
+ with pytest.raises(httpx.ReadError):
+ for _ in stream:
+ pass
+ span = one(spans)
+ assert span.status.status_code == StatusCode.ERROR
+ assert attrs(span)["error.type"] == "ReadError"
+ assert "SECRETDETAIL" not in blob(span)
+
+
+# --- completions -------------------------------------------------------------------------------
+
+def test_completions_sync_async_and_stream(spans):
+ handler = ok_handler(completion_body("done"))
+ client = make_client(handler)
+ client.completions.create(model="palmyra-x-003-instruct", prompt="Say done", max_tokens=5, temperature=0.1)
+ span = spans.get_finished_spans()[0]
+ a = attrs(span)
+ assert span.name == "writer.text_completion"
+ assert a["gen_ai.span.kind"] == "LLM" and a["gen_ai.provider.name"] == "writer"
+ assert a["gen_ai.request.model"] == "palmyra-x-003-instruct"
+ assert a["gen_ai.request.max_tokens"] == 5
+ assert a["input.value"] == "Say done"
+ assert a["output.value"] == "done"
+
+ spans.clear()
+ aclient = make_async_client(ok_handler(completion_body("async done")))
+ asyncio.run(aclient.completions.create(model="m", prompt="p"))
+ assert attrs(one(spans))["output.value"] == "async done"
+
+ spans.clear()
+ stream_h = lambda request: sse({"value": "he"}, {"value": "llo"})
+ for _ in make_client(stream_h).completions.create(model="m", prompt="p", stream=True):
+ pass
+ assert attrs(one(spans))["output.value"] == "hello"
+
+
+# --- errors ------------------------------------------------------------------------------------
+
+@pytest.mark.parametrize("status, error_class", [(429, "RateLimitError"), (500, "InternalServerError"), (401, "AuthenticationError")])
+def test_status_errors_are_reraised_unchanged(spans, status, error_class):
+ handler = lambda request: httpx.Response(status, json={"message": "failure with SECRETDETAIL"})
+ with pytest.raises(writerai.APIStatusError) as raised:
+ make_client(handler).chat.chat(messages=MESSAGES, model="m")
+ assert type(raised.value).__name__ == error_class
+ span = one(spans)
+ assert span.status.status_code == StatusCode.ERROR
+ assert attrs(span)["error.type"] == error_class
+ assert "SECRETDETAIL" not in blob(span)
+ assert span.end_time is not None
+
+
+def test_connection_error_and_async_error(spans):
+ def boom(request):
+ raise httpx.ConnectError("no route SECRETDETAIL")
+
+ with pytest.raises(writerai.APIConnectionError):
+ make_client(boom).completions.create(model="m", prompt="p")
+ assert attrs(one(spans))["error.type"] == "APIConnectionError"
+
+ spans.clear()
+ handler = lambda request: httpx.Response(500, json={"message": "x"})
+ with pytest.raises(writerai.InternalServerError):
+ asyncio.run(make_async_client(handler).chat.chat(messages=MESSAGES, model="m"))
+ assert attrs(one(spans))["error.type"] == "InternalServerError"
+
+
+def test_invalid_arguments_still_raise_sdk_errors(spans):
+ with pytest.raises(TypeError):
+ make_client(ok_handler(chat_body())).chat.chat(messages=MESSAGES) # model is required
+ assert one(spans).status.status_code == StatusCode.ERROR
+
+
+# --- privacy -----------------------------------------------------------------------------------
+
+@pytest.mark.parametrize("config, hidden, visible", [
+ (TraceConfig(hide_inputs=True), "TOPSECRET-IN", "TOPSECRET-OUT"),
+ (TraceConfig(hide_outputs=True), "TOPSECRET-OUT", "TOPSECRET-IN"),
+ (TraceConfig(hide_input_messages=True), "TOPSECRET-IN", "TOPSECRET-OUT"),
+ (TraceConfig(hide_output_text=True), "TOPSECRET-OUT", "TOPSECRET-IN"),
+])
+def test_hidden_content_is_not_exported(instrument, config, hidden, visible):
+ spans = instrument(config=config)
+ make_client(ok_handler(chat_body("TOPSECRET-OUT"))).chat.chat(
+ messages=[{"role": "user", "content": "TOPSECRET-IN"}], model="m"
+ )
+ span = one(spans)
+ assert hidden not in blob(span)
+ assert visible in blob(span)
+ assert attrs(span)["gen_ai.request.model"] == "m"
+ assert attrs(span)["gen_ai.usage.total_tokens"] == 5
+
+
+def test_hidden_completion_prompt(instrument):
+ spans = instrument(config=TraceConfig(hide_inputs=True, hide_outputs=True))
+ make_client(ok_handler(completion_body("TOPSECRET-OUT"))).completions.create(model="m", prompt="TOPSECRET-IN")
+ assert "TOPSECRET" not in blob(one(spans))
+
+
+# --- outermost call only / lifecycle -----------------------------------------------------------
+
+def test_raw_and_streaming_response_variants_trace_once_and_still_parse(spans):
+ client = make_client(ok_handler(chat_body()))
+ raw = client.chat.with_raw_response.chat(messages=MESSAGES, model="m")
+ assert raw.parse().choices[0].message.content == "Tokyo"
+ assert len(spans.get_finished_spans()) == 1
+ spans.clear()
+ with client.chat.with_streaming_response.chat(messages=MESSAGES, model="m") as response:
+ assert response.parse().choices[0].message.content == "Tokyo"
+ assert len(spans.get_finished_spans()) == 1
+
+
+def test_suppressed_instrumentation_creates_no_span(spans):
+ token = context_api.attach(context_api.set_value(_SUPPRESS_INSTRUMENTATION_KEY, True))
+ try:
+ make_client(ok_handler(chat_body())).chat.chat(messages=MESSAGES, model="m")
+ finally:
+ context_api.detach(token)
+ assert spans.get_finished_spans() == ()
+
+
+def test_uninstrument_restores_the_sdk_and_can_reinstrument(provider):
+ from traceai_writer import WriterInstrumentor
+
+ original = writerai.resources.chat.ChatResource.__dict__["chat"]
+ instrumentor = WriterInstrumentor()
+ instrumentor.instrument(tracer_provider=provider)
+ assert writerai.resources.chat.ChatResource.__dict__["chat"] is not original
+ instrumentor.uninstrument()
+ assert writerai.resources.chat.ChatResource.__dict__["chat"] is original
+ make_client(ok_handler(chat_body())).chat.chat(messages=MESSAGES, model="m")
+ assert provider.exporter.get_finished_spans() == ()
+ instrumentor.instrument(tracer_provider=provider)
+ make_client(ok_handler(chat_body())).chat.chat(messages=MESSAGES, model="m")
+ instrumentor.uninstrument()
+ assert len(provider.exporter.get_finished_spans()) == 1
+
+
+def test_dependency_declaration():
+ from traceai_writer import WriterInstrumentor, __version__
+
+ assert any(dep.startswith("writer-sdk") for dep in WriterInstrumentor().instrumentation_dependencies())
+ assert __version__
diff --git a/python/tests/test_auto_entrypoints.py b/python/tests/test_auto_entrypoints.py
index 7be4a4b7d..494e11f00 100644
--- a/python/tests/test_auto_entrypoints.py
+++ b/python/tests/test_auto_entrypoints.py
@@ -372,6 +372,16 @@ def test_registry_enables_sagemaker_on_boto3_next_to_bedrock():
assert bedrock.targets == sagemaker.targets
+def test_registry_enables_writer_on_the_writerai_import():
+ integration = _registry.find("writer")
+ assert integration is not None
+ assert integration.kind == _registry.INSTRUMENTOR
+ assert integration.targets == ("writerai",)
+ assert (integration.package, integration.module, integration.entry) == (
+ "traceAI-writer", "traceai_writer", "WriterInstrumentor")
+ assert integration.default
+
+
@pytest.mark.parametrize("name, target, package", [
("agentscope", "agentscope", "traceAI-agentscope"),
("temporal", "temporalio", "traceAI-temporal"),
From e61a49e9706b06acf9b6ad56353224ce41af5459 Mon Sep 17 00:00:00 2001
From: nik13 <8347993+nik13@users.noreply.github.com>
Date: Thu, 19 Nov 2026 03:26:58 +0000
Subject: [PATCH 6/8] feat(writer): trace Writer SDK chat, completions, tool
calls and streams; register zero-code auto-instrumentation
---
.github/release/config.json | 3 +-
.github/workflows/writer-python-tests.yml | 60 +++
.release-please-manifest.json | 1 +
docs/python/auto-instrumentation.md | 1 +
python/README.md | 1 +
python/fi_instrumentation/auto/_registry.py | 1 +
python/frameworks/writer/CHANGELOG.md | 5 +
python/frameworks/writer/README.md | 41 ++
python/frameworks/writer/pyproject.toml | 26 ++
python/frameworks/writer/tests/test_writer.py | 2 +
.../writer/traceai_writer/__init__.py | 393 ++++++++++++++++++
.../writer/traceai_writer/version.py | 1 +
release-please-config.json | 8 +
13 files changed, 542 insertions(+), 1 deletion(-)
create mode 100644 .github/workflows/writer-python-tests.yml
create mode 100644 python/frameworks/writer/CHANGELOG.md
create mode 100644 python/frameworks/writer/README.md
create mode 100644 python/frameworks/writer/pyproject.toml
create mode 100644 python/frameworks/writer/traceai_writer/__init__.py
create mode 100644 python/frameworks/writer/traceai_writer/version.py
diff --git a/.github/release/config.json b/.github/release/config.json
index 7e97bf100..c6173b23f 100644
--- a/.github/release/config.json
+++ b/.github/release/config.json
@@ -30,7 +30,8 @@
"traceAI-openrouter",
"traceAI-sagemaker",
"traceAI-marqo",
- "traceAI-watsonx"
+ "traceAI-watsonx",
+ "traceAI-writer"
],
"_hold_artifacts_comment": "Reactor modules that the Java release builds and tests but does not deploy, with the reason. A held module's release profile sets skipPublishing, and the release plan leaves it out of the Central check, so the rest of the reactor still releases. To publish one, remove its entry here and its skipPublishing setting.",
"hold_artifacts": {}
diff --git a/.github/workflows/writer-python-tests.yml b/.github/workflows/writer-python-tests.yml
new file mode 100644
index 000000000..3a157a1b9
--- /dev/null
+++ b/.github/workflows/writer-python-tests.yml
@@ -0,0 +1,60 @@
+name: Writer Python SDK tests
+
+on:
+ pull_request:
+ branches: [main, dev]
+ paths:
+ - 'python/frameworks/writer/**'
+ - 'python/fi_instrumentation/**'
+ - '.github/workflows/writer-python-tests.yml'
+ push:
+ branches: [dev]
+ paths:
+ - 'python/frameworks/writer/**'
+ - 'python/fi_instrumentation/**'
+ - '.github/workflows/writer-python-tests.yml'
+
+permissions:
+ contents: read
+
+jobs:
+ sdk-range:
+ runs-on: ubuntu-24.04
+ timeout-minutes: 15
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - python: '3.10'
+ writer: '2.0.0'
+ artifact: wheel
+ - python: '3.13'
+ writer: 'latest'
+ artifact: sdist
+ steps:
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+ with:
+ persist-credentials: false
+ - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ with:
+ python-version: ${{ matrix.python }}
+ - name: Build and install the package
+ shell: bash
+ env:
+ ARTIFACT: ${{ matrix.artifact }}
+ WRITER_VERSION: ${{ matrix.writer }}
+ run: |
+ python -m venv .writer-test-venv
+ .writer-test-venv/bin/pip install build
+ .writer-test-venv/bin/python -m build "--$ARTIFACT" python/frameworks/writer
+ if [ "$ARTIFACT" = wheel ]; then ext='*.whl'; else ext='*.tar.gz'; fi
+ if [ "$WRITER_VERSION" = latest ]; then sdk='writer-sdk'; else sdk="writer-sdk==$WRITER_VERSION"; fi
+ .writer-test-venv/bin/pip install python/frameworks/writer/dist/$ext "$sdk" pytest pytest-asyncio
+ - name: Run offline tests without external traffic
+ shell: bash
+ run: |
+ sudo unshare --net bash -c '
+ set -e
+ ip link set lo up
+ "$1" -I -m pytest python/frameworks/writer/tests -q -p no:cacheprovider
+ ' _ "$PWD/.writer-test-venv/bin/python"
diff --git a/.release-please-manifest.json b/.release-please-manifest.json
index 98469dcfc..1f33af287 100644
--- a/.release-please-manifest.json
+++ b/.release-please-manifest.json
@@ -161,6 +161,7 @@
"python/frameworks/sagemaker": "0.1.0",
"python/frameworks/marqo": "0.1.0",
"python/frameworks/watsonx": "0.1.0",
+ "python/frameworks/writer": "0.1.0",
"go/langchaingo": "0.1.0",
"ruby": "0.1.0",
"ruby/roast": "0.1.0",
diff --git a/docs/python/auto-instrumentation.md b/docs/python/auto-instrumentation.md
index 1350b1a78..28950127b 100644
--- a/docs/python/auto-instrumentation.md
+++ b/docs/python/auto-instrumentation.md
@@ -98,6 +98,7 @@ Each integration reports one of these statuses:
| `transformers` | `transformers` | `traceAI-transformers` | Automatic |
| `litellm` | `litellm` | `traceAI-litellm` | Automatic |
| `openrouter` | `openrouter` | `traceAI-openrouter` | Automatic |
+| `writer` | `writerai` | `traceAI-writer` | Automatic |
| `portkey` | `portkey-ai` | `traceAI-portkey` | Automatic |
| `watsonx` | `ibm_watsonx_ai` | `traceAI-watsonx` | Automatic |
| `replicate` | `replicate` | `traceAI-replicate` | Automatic |
diff --git a/python/README.md b/python/README.md
index 874c04059..3645f62f4 100644
--- a/python/README.md
+++ b/python/README.md
@@ -223,6 +223,7 @@ Each package below links to its own installation instructions, supported SDK ver
| `traceAI-vllm` | [vllm](frameworks/vllm/) |
| `traceAI-voyage` | [voyage](frameworks/voyage/) |
| `traceAI-weaviate` | [weaviate](frameworks/weaviate/) |
+| `traceAI-writer` | [writer](frameworks/writer/) |
| `traceAI-xai` | [xai](frameworks/xai/) |
diff --git a/python/fi_instrumentation/auto/_registry.py b/python/fi_instrumentation/auto/_registry.py
index f893db9a9..90181df29 100644
--- a/python/fi_instrumentation/auto/_registry.py
+++ b/python/fi_instrumentation/auto/_registry.py
@@ -68,6 +68,7 @@ def _covered_by_openai(provider: str) -> str:
Integration("transformers", ("transformers",), "traceAI-transformers", "traceai_transformers", "TransformersInstrumentor"),
Integration("litellm", ("litellm",), "traceAI-litellm", "traceai_litellm", "LiteLLMInstrumentor", category="gateway"),
Integration("openrouter", ("openrouter",), "traceAI-openrouter", "traceai_openrouter", "OpenRouterInstrumentor", category="gateway"),
+ Integration("writer", ("writerai",), "traceAI-writer", "traceai_writer", "WriterInstrumentor"),
Integration("portkey", ("portkey_ai",), "traceAI-portkey", "traceai_portkey", "PortkeyInstrumentor", category="gateway"),
Integration("watsonx", ("ibm_watsonx_ai",), "traceAI-watsonx", "traceai_watsonx", "WatsonxInstrumentor"),
Integration("replicate", ("replicate",), "traceAI-replicate", "traceai_replicate", "ReplicateInstrumentor"),
diff --git a/python/frameworks/writer/CHANGELOG.md b/python/frameworks/writer/CHANGELOG.md
new file mode 100644
index 000000000..95e719de4
--- /dev/null
+++ b/python/frameworks/writer/CHANGELOG.md
@@ -0,0 +1,5 @@
+# Changelog
+
+## 0.1.0
+
+- Instrument the Writer Python SDK (`writer-sdk`): chat (including tool calling and streaming) and text completions, synchronous and asynchronous.
diff --git a/python/frameworks/writer/README.md b/python/frameworks/writer/README.md
new file mode 100644
index 000000000..ef3844d2d
--- /dev/null
+++ b/python/frameworks/writer/README.md
@@ -0,0 +1,41 @@
+# traceAI-writer
+
+OpenTelemetry instrumentation for the [Writer Python SDK](https://github.com/writer/writer-python)
+(`writer-sdk`, import `writerai`).
+
+```bash
+pip install traceAI-writer
+```
+
+```python
+from fi_instrumentation import register
+from traceai_writer import WriterInstrumentor
+from writerai import Writer
+
+WriterInstrumentor().instrument(tracer_provider=register(project_name="my-project"))
+
+client = Writer()
+client.chat.chat(model="palmyra-x5", messages=[{"role": "user", "content": "Hello"}])
+```
+
+With `fi-instrumentation-otel` auto-instrumentation the `writer` integration is enabled whenever
+`writerai` is installed.
+
+## What is traced
+
+`chat.chat` (`writer.chat`) and `completions.create` (`writer.text_completion`), sync and async,
+streaming or not. Each is an `LLM` span with `gen_ai.provider.name = writer`, the request model,
+temperature / top_p / max_tokens / stop, the response model and id, finish reasons and token usage
+when the response carries it. Tool calls the model makes are recorded on the output message.
+
+* Only the outermost SDK call is traced: `chat.stream()`, `with_raw_response` and
+ `with_streaming_response` produce one span, not several.
+* Streams end on exhaustion, error or `close()`; an early close is marked
+ `writer.stream.incomplete`.
+* Exceptions are re-raised unchanged and recorded as `ERROR` with `error.type` (the class name);
+ the message is never exported.
+* Inputs and outputs follow the `TraceConfig` hide settings. Only plain-text conversations are
+ captured as `input.value`; multimodal parts, and Knowledge Graph / web-search source documents
+ returned with a response, are never exported.
+
+Supported SDK range: `writer-sdk >= 2.0.0, < 4`.
diff --git a/python/frameworks/writer/pyproject.toml b/python/frameworks/writer/pyproject.toml
new file mode 100644
index 000000000..a9bf091cd
--- /dev/null
+++ b/python/frameworks/writer/pyproject.toml
@@ -0,0 +1,26 @@
+[tool.poetry]
+name = "traceAI-writer"
+version = "0.1.0"
+description = "OpenTelemetry instrumentation for the Writer Python SDK"
+authors = ["Future AGI "]
+readme = "README.md"
+license = "Apache-2.0"
+packages = [{ include = "traceai_writer" }]
+classifiers = ["Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Framework :: OpenTelemetry :: Instrumentations"]
+
+[tool.poetry.dependencies]
+python = ">=3.10,<3.14"
+fi-instrumentation-otel = ">=1.1.0"
+writer-sdk = ">=2.0.0,<4"
+wrapt = ">=1.15.0"
+
+[tool.poetry.group.dev.dependencies]
+pytest = ">=7"
+pytest-asyncio = ">=0.23"
+
+[tool.poetry.urls]
+Repository = "https://github.com/future-agi/traceAI/tree/dev/python/frameworks/writer"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
diff --git a/python/frameworks/writer/tests/test_writer.py b/python/frameworks/writer/tests/test_writer.py
index b18d3d2c6..69e67969b 100644
--- a/python/frameworks/writer/tests/test_writer.py
+++ b/python/frameworks/writer/tests/test_writer.py
@@ -207,6 +207,8 @@ def test_streamed_tool_call_deltas_are_assembled(spans):
def test_chat_stream_helper_produces_exactly_one_span(spans):
+ if not hasattr(writerai.resources.chat.ChatResource, "stream"):
+ pytest.skip("chat.stream() helper was added after writer-sdk 2.0.0")
with make_client(stream_handler()).chat.stream(messages=MESSAGES, model="palmyra-x5") as stream:
for _ in stream:
pass
diff --git a/python/frameworks/writer/traceai_writer/__init__.py b/python/frameworks/writer/traceai_writer/__init__.py
new file mode 100644
index 000000000..229c189af
--- /dev/null
+++ b/python/frameworks/writer/traceai_writer/__init__.py
@@ -0,0 +1,393 @@
+"""Trace the Writer Python SDK (``writer-sdk``, import ``writerai``).
+
+Chat (including tool calling and streaming) and legacy text completions are traced, sync and
+async. Only the outermost SDK call produces a span: ``client.chat.stream()`` and the
+``with_raw_response`` / ``with_streaming_response`` variants all end up in the one wrapped
+method. Do not instrument HTTPX alongside this instrumentor for the same requests.
+"""
+from __future__ import annotations
+
+import json
+import logging
+from importlib import import_module
+from typing import Any, Collection, Dict, List, Optional
+
+from fi_instrumentation import FITracer, TraceConfig
+from fi_instrumentation.fi_types import FiSpanKindValues, SpanAttributes as A
+from opentelemetry import trace
+from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore
+from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, get_value
+from opentelemetry.trace import Status, StatusCode
+from wrapt import wrap_function_wrapper
+
+from .version import __version__
+
+logger = logging.getLogger(__name__)
+logger.addHandler(logging.NullHandler())
+
+_PROVIDER = "writer"
+_LIMIT = 32768
+_OUT = f"{A.GEN_AI_OUTPUT_MESSAGES}.0.message"
+
+# (module, sync class, async class, method, operation)
+_ENDPOINTS = (
+ ("writerai.resources.chat", "ChatResource", "AsyncChatResource", "chat", "chat"),
+ ("writerai.resources.completions", "CompletionsResource", "AsyncCompletionsResource", "create", "text_completion"),
+)
+_PARAMETERS = (
+ ("temperature", A.GEN_AI_REQUEST_TEMPERATURE),
+ ("top_p", A.GEN_AI_REQUEST_TOP_P),
+ ("max_tokens", A.GEN_AI_REQUEST_MAX_TOKENS),
+)
+
+
+def _field(value: Any, name: str) -> Any:
+ if isinstance(value, dict):
+ return value.get(name)
+ return getattr(value, name, None)
+
+
+def _count(value: Any) -> Optional[int]:
+ if isinstance(value, int) and not isinstance(value, bool) and value >= 0:
+ return value
+ return None
+
+
+def _text(value: Any) -> Optional[str]:
+ return value if isinstance(value, str) and value else None
+
+
+class _Recorder:
+ """Folds a response, or the chunks of a streamed response, into span attributes."""
+
+ def __init__(self, operation: str, capture_output: bool) -> None:
+ self.operation = operation
+ self.capture_output = capture_output
+ self.model: Optional[str] = None
+ self.response_id: Optional[str] = None
+ self.finish: List[str] = []
+ self.usage: Dict[str, int] = {}
+ self.pieces: List[str] = []
+ self.length = 0
+ self.tools: Dict[int, Dict[str, Any]] = {}
+
+ def _add(self, text: Optional[str]) -> None:
+ if text and self.length < _LIMIT:
+ text = text[: _LIMIT - self.length]
+ self.pieces.append(text)
+ self.length += len(text)
+
+ def _tool_calls(self, calls: Any) -> None:
+ if not isinstance(calls, (list, tuple)):
+ return
+ for position, call in enumerate(calls):
+ index = _count(_field(call, "index"))
+ slot = self.tools.setdefault(position if index is None else index, {"args": []})
+ if _text(_field(call, "id")):
+ slot["id"] = _field(call, "id")
+ function = _field(call, "function")
+ if _text(_field(function, "name")):
+ slot["name"] = _field(function, "name")
+ arguments = _field(function, "arguments")
+ if isinstance(arguments, str):
+ slot["args"].append(arguments)
+
+ def observe(self, value: Any, chunk: bool = False) -> None:
+ """Fold a ChatCompletion, Completion or a streamed chunk into the recorder."""
+ if _text(_field(value, "model")):
+ self.model = _field(value, "model")
+ if _text(_field(value, "id")):
+ self.response_id = _field(value, "id")
+ usage = _field(value, "usage")
+ if usage is not None:
+ for key, names in (
+ ("input", ("prompt_tokens", "input_tokens")),
+ ("output", ("completion_tokens", "output_tokens")),
+ ("total", ("total_tokens",)),
+ ):
+ for name in names:
+ number = _count(_field(usage, name))
+ if number is not None:
+ self.usage[key] = number
+ break
+ choices = _field(value, "choices")
+ if not isinstance(choices, (list, tuple)):
+ self._add(_text(_field(value, "value"))) # legacy completion stream chunk
+ return
+ for position, choice in enumerate(choices):
+ reason = _field(choice, "finish_reason")
+ if _text(reason) and reason not in self.finish:
+ self.finish.append(reason)
+ if position:
+ continue # content comes from the first choice
+ if self.operation == "text_completion":
+ self._add(_text(_field(choice, "text")))
+ continue
+ holder = _field(choice, "delta") if chunk else _field(choice, "message")
+ if holder is None:
+ holder = _field(choice, "delta") or _field(choice, "message")
+ if holder is not None:
+ self._add(_text(_field(holder, "content")))
+ self._tool_calls(_field(holder, "tool_calls"))
+
+ def apply(self, span: Any) -> None:
+ if self.model:
+ span.set_attribute(A.GEN_AI_RESPONSE_MODEL, self.model)
+ if self.response_id:
+ span.set_attribute(A.GEN_AI_RESPONSE_ID, self.response_id)
+ if self.finish:
+ span.set_attribute(A.GEN_AI_RESPONSE_FINISH_REASONS, self.finish)
+ total = self.usage.get("total")
+ if total is None and "input" in self.usage and "output" in self.usage:
+ total = self.usage["input"] + self.usage["output"]
+ for attr, value in (
+ (A.GEN_AI_USAGE_INPUT_TOKENS, self.usage.get("input")),
+ (A.GEN_AI_USAGE_OUTPUT_TOKENS, self.usage.get("output")),
+ (A.GEN_AI_USAGE_TOTAL_TOKENS, total),
+ ):
+ if value is not None:
+ span.set_attribute(attr, value)
+ for position, slot in sorted(self.tools.items()):
+ prefix = f"{_OUT}.tool_calls.{position}.tool_call"
+ if slot.get("id"):
+ span.set_attribute(f"{prefix}.id", slot["id"])
+ if slot.get("name"):
+ span.set_attribute(f"{prefix}.function.name", slot["name"])
+ arguments = "".join(slot["args"])
+ if self.capture_output and arguments:
+ span.set_attribute(f"{prefix}.function.arguments", arguments[:_LIMIT])
+ if self.capture_output and self.pieces:
+ span.set_attribute(A.OUTPUT_VALUE, "".join(self.pieces))
+
+
+def _request(span: Any, operation: str, kwargs: Dict[str, Any], capture_input: bool) -> None:
+ for name, attr in _PARAMETERS:
+ value = kwargs.get(name)
+ if isinstance(value, (int, float)) and not isinstance(value, bool):
+ span.set_attribute(attr, value)
+ stop = kwargs.get("stop")
+ if isinstance(stop, str):
+ stop = [stop]
+ if isinstance(stop, (list, tuple)) and stop and all(isinstance(s, str) for s in stop):
+ span.set_attribute(A.GEN_AI_REQUEST_STOP_SEQUENCES, list(stop))
+ if not capture_input:
+ return
+ if operation == "text_completion":
+ prompt = kwargs.get("prompt")
+ if isinstance(prompt, str) and "data:image/" not in prompt.lower():
+ span.set_attribute(A.INPUT_VALUE, prompt[:_LIMIT])
+ span.set_attribute(A.INPUT_MIME_TYPE, "text/plain")
+ return
+ messages = kwargs.get("messages")
+ # Only plain text conversations are serialized wholesale: input.value cannot tell text
+ # from an inline image, document or tool payload inside a structured content part.
+ if not isinstance(messages, (list, tuple)) or any(
+ not isinstance(m, dict) or not isinstance(m.get("role"), str) or not isinstance(m.get("content"), str)
+ for m in messages
+ ):
+ return
+ plain = [{"role": m["role"], "content": m["content"]} for m in messages]
+ serialized = json.dumps(plain)
+ if "data:image/" in serialized.lower():
+ return
+ span.set_attribute(A.INPUT_VALUE, serialized[:_LIMIT])
+ span.set_attribute(A.INPUT_MIME_TYPE, "application/json")
+
+
+def _failure(span: Any, exc: BaseException) -> None:
+ # Exception text can contain the request, response body or credentials: class name only.
+ span.set_attribute("error.type", type(exc).__name__)
+ span.set_status(Status(StatusCode.ERROR))
+ span.end()
+
+
+class _Stream:
+ """Proxy for the SDK's Stream / AsyncStream; chunks are forwarded unchanged."""
+
+ def __init__(self, source: Any, span: Any, recorder: _Recorder) -> None:
+ self._source, self._span, self._recorder = source, span, recorder
+ self._ended = False
+
+ def __getattr__(self, name: str) -> Any:
+ return getattr(self._source, name)
+
+ def _finish(self, complete: bool = True) -> None:
+ if self._ended:
+ return
+ self._ended = True
+ try:
+ if not complete:
+ self._span.set_attribute("writer.stream.incomplete", True)
+ self._recorder.apply(self._span)
+ except Exception:
+ logger.debug("writer stream finalisation failed", exc_info=True)
+ finally:
+ self._span.end()
+
+ def _fail(self, exc: BaseException) -> None:
+ if not self._ended:
+ self._ended = True
+ _failure(self._span, exc)
+
+ def _observe(self, chunk: Any) -> None:
+ try:
+ self._recorder.observe(chunk, chunk=True)
+ except Exception:
+ logger.debug("writer stream capture failed", exc_info=True)
+
+ def __iter__(self) -> "_Stream":
+ return self
+
+ def __next__(self) -> Any:
+ try:
+ with trace.use_span(self._span, end_on_exit=False, record_exception=False, set_status_on_exception=False):
+ chunk = next(self._source)
+ except StopIteration:
+ self._finish()
+ raise
+ except BaseException as exc:
+ self._fail(exc)
+ raise
+ self._observe(chunk)
+ return chunk
+
+ def __aiter__(self) -> "_Stream":
+ return self
+
+ async def __anext__(self) -> Any:
+ try:
+ with trace.use_span(self._span, end_on_exit=False, record_exception=False, set_status_on_exception=False):
+ chunk = await self._source.__anext__()
+ except StopAsyncIteration:
+ self._finish()
+ raise
+ except BaseException as exc:
+ self._fail(exc)
+ raise
+ self._observe(chunk)
+ return chunk
+
+ def close(self) -> Any:
+ try:
+ return self._source.close()
+ finally:
+ self._finish(complete=False)
+
+ def __enter__(self) -> "_Stream":
+ self._source.__enter__()
+ return self
+
+ def __exit__(self, *exc_info: Any) -> Any:
+ try:
+ return self._source.__exit__(*exc_info)
+ finally:
+ self._finish(complete=False)
+
+ async def __aenter__(self) -> "_Stream":
+ await self._source.__aenter__()
+ return self
+
+ async def __aexit__(self, *exc_info: Any) -> Any:
+ try:
+ return await self._source.__aexit__(*exc_info)
+ finally:
+ self._finish(complete=False)
+
+
+class _AsyncStream(_Stream):
+ async def close(self) -> Any: # type: ignore[override]
+ try:
+ return await self._source.close()
+ finally:
+ self._finish(complete=False)
+
+
+def _start(tracer: Any, config: TraceConfig, operation: str, kwargs: Dict[str, Any]) -> Any:
+ attrs: Dict[str, Any] = {A.GEN_AI_PROVIDER_NAME: _PROVIDER, A.GEN_AI_OPERATION_NAME: operation}
+ model = kwargs.get("model")
+ if isinstance(model, str) and model:
+ attrs[A.GEN_AI_REQUEST_MODEL] = model
+ span = tracer.start_span(f"writer.{operation}", attributes=attrs, fi_span_kind=FiSpanKindValues.LLM)
+ try:
+ capture_input = not (
+ config.hide_inputs or config.hide_input_text or config.hide_input_messages or config.hide_input_images
+ )
+ _request(span, operation, kwargs, capture_input)
+ except Exception:
+ logger.debug("writer request capture failed", exc_info=True)
+ return span
+
+
+def _complete(span: Any, result: Any, operation: str, config: TraceConfig) -> Any:
+ capture_output = not (config.hide_outputs or config.hide_output_text or config.hide_output_messages)
+ recorder = _Recorder(operation, capture_output)
+ try:
+ if hasattr(result, "__anext__"):
+ return _AsyncStream(result, span, recorder)
+ if hasattr(result, "__next__"):
+ return _Stream(result, span, recorder)
+ if hasattr(result, "choices"):
+ recorder.observe(result)
+ recorder.apply(span)
+ # Anything else is a raw or streaming-response wrapper: its body is not ours to read.
+ except Exception:
+ logger.debug("writer response capture failed", exc_info=True)
+ span.end()
+ return result
+
+
+def _wrap(tracer: Any, config: TraceConfig, operation: str, is_async: bool) -> Any:
+ if is_async:
+ async def wrapped(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any:
+ if get_value(_SUPPRESS_INSTRUMENTATION_KEY):
+ return await wrapped(*args, **kwargs)
+ span = _start(tracer, config, operation, kwargs)
+ try:
+ with trace.use_span(span, end_on_exit=False, record_exception=False, set_status_on_exception=False):
+ result = await wrapped(*args, **kwargs)
+ except BaseException as exc:
+ _failure(span, exc)
+ raise
+ return _complete(span, result, operation, config)
+ else:
+ def wrapped(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: # type: ignore[misc]
+ if get_value(_SUPPRESS_INSTRUMENTATION_KEY):
+ return wrapped(*args, **kwargs)
+ span = _start(tracer, config, operation, kwargs)
+ try:
+ with trace.use_span(span, end_on_exit=False, record_exception=False, set_status_on_exception=False):
+ result = wrapped(*args, **kwargs)
+ except BaseException as exc:
+ _failure(span, exc)
+ raise
+ return _complete(span, result, operation, config)
+ return wrapped
+
+
+class WriterInstrumentor(BaseInstrumentor): # type: ignore[misc]
+ """Instrument Writer SDK chat and completions calls (sync and async)."""
+
+ def instrumentation_dependencies(self) -> Collection[str]:
+ return ("writer-sdk >= 2.0.0, < 4",)
+
+ def _instrument(self, **kwargs: Any) -> None:
+ config = kwargs.get("config") or TraceConfig()
+ if not isinstance(config, TraceConfig):
+ raise TypeError("config must be a TraceConfig")
+ tracer = FITracer(trace.get_tracer(__name__, __version__, kwargs.get("tracer_provider")), config)
+ self._originals: List[Any] = []
+ for module, sync_cls, async_cls, method, operation in _ENDPOINTS:
+ mod = import_module(module)
+ for cls_name, is_async in ((sync_cls, False), (async_cls, True)):
+ cls = getattr(mod, cls_name)
+ original = cls.__dict__[method]
+ wrap_function_wrapper(module, f"{cls_name}.{method}", _wrap(tracer, config, operation, is_async))
+ self._originals.append((cls, method, original))
+
+ def _uninstrument(self, **kwargs: Any) -> None:
+ for cls, name, original in getattr(self, "_originals", []):
+ setattr(cls, name, original)
+ self._originals = []
+
+
+__all__ = ["WriterInstrumentor", "__version__"]
diff --git a/python/frameworks/writer/traceai_writer/version.py b/python/frameworks/writer/traceai_writer/version.py
new file mode 100644
index 000000000..75716eb9a
--- /dev/null
+++ b/python/frameworks/writer/traceai_writer/version.py
@@ -0,0 +1 @@
+__version__ = "0.1.0" # x-release-please-version
diff --git a/release-please-config.json b/release-please-config.json
index c400ac422..d5d808521 100644
--- a/release-please-config.json
+++ b/release-please-config.json
@@ -1370,6 +1370,14 @@
"traceai_sagemaker/version.py"
]
},
+ "python/frameworks/writer": {
+ "release-type": "python",
+ "component": "traceai-writer",
+ "package-name": "traceAI-writer",
+ "extra-files": [
+ "traceai_writer/version.py"
+ ]
+ },
"go/langchaingo": {
"release-type": "go",
"component": "go/langchaingo",
From bdcd99d7141e1508e835f12cc16a84baf4f48a31 Mon Sep 17 00:00:00 2001
From: nik13 <8347993+nik13@users.noreply.github.com>
Date: Thu, 19 Nov 2026 03:39:43 +0000
Subject: [PATCH 7/8] test(writer): cover chat.stream helper left early and
abandoned streams
---
python/frameworks/writer/tests/test_writer.py | 64 +++++++++++++++++++
1 file changed, 64 insertions(+)
diff --git a/python/frameworks/writer/tests/test_writer.py b/python/frameworks/writer/tests/test_writer.py
index 69e67969b..bdfb6c1e4 100644
--- a/python/frameworks/writer/tests/test_writer.py
+++ b/python/frameworks/writer/tests/test_writer.py
@@ -376,3 +376,67 @@ def test_dependency_declaration():
assert any(dep.startswith("writer-sdk") for dep in WriterInstrumentor().instrumentation_dependencies())
assert __version__
+
+
+# --- early exit from the chat.stream() helper --------------------------------------------------
+
+def _need_helper():
+ if not hasattr(writerai.resources.chat.ChatResource, "stream"):
+ pytest.skip("chat.stream() helper was added after writer-sdk 2.0.0")
+
+
+def test_chat_stream_helper_left_early_ends_its_span_once(spans):
+ _need_helper()
+ with make_client(stream_handler()).chat.stream(messages=MESSAGES, model="palmyra-x5") as stream:
+ for _event in stream:
+ break
+ span = one(spans)
+ assert span.name == "writer.chat"
+ assert attrs(span)["writer.stream.incomplete"] is True
+
+
+def test_chat_stream_helper_explicit_close_ends_its_span_once(spans):
+ _need_helper()
+ manager = make_client(stream_handler()).chat.stream(messages=MESSAGES, model="m")
+ stream = manager.__enter__()
+ next(iter(stream))
+ assert spans.get_finished_spans() == ()
+ stream.close()
+ stream.close()
+ manager.__exit__(None, None, None)
+ assert attrs(one(spans))["writer.stream.incomplete"] is True
+
+
+def test_chat_stream_helper_read_to_the_end_is_not_incomplete(spans):
+ _need_helper()
+ with make_client(stream_handler()).chat.stream(messages=MESSAGES, model="m") as stream:
+ for _event in stream:
+ pass
+ span = one(spans)
+ assert "writer.stream.incomplete" not in attrs(span)
+ assert attrs(span)["output.value"] == "Tokyo"
+
+
+def test_async_chat_stream_helper_left_early_ends_its_span_once(spans):
+ _need_helper()
+ client = make_async_client(stream_handler())
+
+ async def go():
+ async with client.chat.stream(messages=MESSAGES, model="palmyra-x5") as stream:
+ async for _event in stream:
+ break
+
+ asyncio.run(go())
+ span = one(spans)
+ assert attrs(span)["writer.stream.incomplete"] is True
+
+
+def test_abandoned_stream_is_ended_when_it_is_collected(spans):
+ import gc
+
+ stream = make_client(stream_handler()).chat.chat(messages=MESSAGES, model="m", stream=True)
+ next(iter(stream))
+ assert spans.get_finished_spans() == ()
+ del stream
+ gc.collect()
+ assert attrs(one(spans))["writer.stream.incomplete"] is True
From 6b9c242540451f9ff10eae148130aab7b3abe4ab Mon Sep 17 00:00:00 2001
From: nik13 <8347993+nik13@users.noreply.github.com>
Date: Thu, 19 Nov 2026 03:40:28 +0000
Subject: [PATCH 8/8] fix(writer): end the span when chat.stream closes its
response or the stream is dropped
---
python/frameworks/writer/README.md | 3 +-
.../writer/traceai_writer/__init__.py | 37 +++++++++++++++++++
2 files changed, 39 insertions(+), 1 deletion(-)
diff --git a/python/frameworks/writer/README.md b/python/frameworks/writer/README.md
index ef3844d2d..d0cffef57 100644
--- a/python/frameworks/writer/README.md
+++ b/python/frameworks/writer/README.md
@@ -31,7 +31,8 @@ when the response carries it. Tool calls the model makes are recorded on the out
* Only the outermost SDK call is traced: `chat.stream()`, `with_raw_response` and
`with_streaming_response` produce one span, not several.
* Streams end on exhaustion, error or `close()`; an early close is marked
- `writer.stream.incomplete`.
+ `writer.stream.incomplete`. This includes leaving a `chat.stream()` helper block early (the
+ helper closes the underlying response) and a stream that is dropped without being closed.
* Exceptions are re-raised unchanged and recorded as `ERROR` with `error.type` (the class name);
the message is never exported.
* Inputs and outputs follow the `TraceConfig` hide settings. Only plain-text conversations are
diff --git a/python/frameworks/writer/traceai_writer/__init__.py b/python/frameworks/writer/traceai_writer/__init__.py
index 229c189af..4b9306340 100644
--- a/python/frameworks/writer/traceai_writer/__init__.py
+++ b/python/frameworks/writer/traceai_writer/__init__.py
@@ -201,6 +201,32 @@ def _failure(span: Any, exc: BaseException) -> None:
span.end()
+class _Response:
+ """The httpx response behind a stream, closed by the SDK's own helpers.
+
+ ``chat.stream()`` keeps ``raw_stream.response`` and closes that, never the stream, so the
+ span has to end when the response is closed too.
+ """
+
+ def __init__(self, response: Any, finish: Any) -> None:
+ self._response, self._finish = response, finish
+
+ def __getattr__(self, name: str) -> Any:
+ return getattr(self._response, name)
+
+ def close(self) -> Any:
+ try:
+ return self._response.close()
+ finally:
+ self._finish(complete=False)
+
+ async def aclose(self) -> Any:
+ try:
+ return await self._response.aclose()
+ finally:
+ self._finish(complete=False)
+
+
class _Stream:
"""Proxy for the SDK's Stream / AsyncStream; chunks are forwarded unchanged."""
@@ -211,6 +237,17 @@ def __init__(self, source: Any, span: Any, recorder: _Recorder) -> None:
def __getattr__(self, name: str) -> Any:
return getattr(self._source, name)
+ @property
+ def response(self) -> Any:
+ return _Response(self._source.response, self._finish)
+
+ def __del__(self) -> None:
+ # A stream dropped without being read or closed must not leak an open span.
+ try:
+ self._finish(complete=False)
+ except Exception:
+ pass
+
def _finish(self, complete: bool = True) -> None:
if self._ended:
return