Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/release/config.json
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@
"traceAI-watsonx",
"traceAI-writer",
"traceAI-alephalpha",
"traceAI-agentspec"
"traceAI-agentspec",
"traceAI-promptflow"
],
"_hold_artifacts_comment": "Reactor modules that the Java release builds and tests but does not deploy, with the reason. A held module's release profile sets skipPublishing, and the release plan leaves it out of the Central check, so the rest of the reactor still releases. To publish one, remove its entry here and its skipPublishing setting.",
"hold_artifacts": {}
Expand Down
60 changes: 60 additions & 0 deletions .github/workflows/promptflow-python-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
name: Prompt flow Python SDK tests

on:
pull_request:
branches: [main, dev]
paths:
- 'python/frameworks/promptflow/**'
- 'python/fi_instrumentation/**'
- '.github/workflows/promptflow-python-tests.yml'
push:
branches: [dev]
paths:
- 'python/frameworks/promptflow/**'
- 'python/fi_instrumentation/**'
- '.github/workflows/promptflow-python-tests.yml'

permissions:
contents: read

jobs:
sdk-range:
runs-on: ubuntu-24.04
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
include:
- python: '3.10'
promptflow: '1.12.0'
artifact: wheel
- python: '3.13'
promptflow: 'latest'
artifact: sdist
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
persist-credentials: false
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: ${{ matrix.python }}
- name: Build and install the package
shell: bash
env:
ARTIFACT: ${{ matrix.artifact }}
PROMPTFLOW_VERSION: ${{ matrix.promptflow }}
run: |
python -m venv .promptflow-test-venv
.promptflow-test-venv/bin/pip install build
.promptflow-test-venv/bin/python -m build "--$ARTIFACT" python/frameworks/promptflow
if [ "$ARTIFACT" = wheel ]; then ext='*.whl'; else ext='*.tar.gz'; fi
if [ "$PROMPTFLOW_VERSION" = latest ]; then sdk='promptflow-tracing promptflow-core'; else sdk="promptflow-tracing==$PROMPTFLOW_VERSION promptflow-core==$PROMPTFLOW_VERSION"; fi
.promptflow-test-venv/bin/pip install python/frameworks/promptflow/dist/$ext $sdk pytest pytest-asyncio openai httpx
- name: Run offline tests without external traffic
shell: bash
run: |
sudo unshare --net bash -c '
set -e
ip link set lo up
"$1" -I -m pytest python/frameworks/promptflow/tests -q -p no:cacheprovider
' _ "$PWD/.promptflow-test-venv/bin/python"
1 change: 1 addition & 0 deletions .release-please-manifest.json
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@
"python/frameworks/writer": "0.1.0",
"python/frameworks/alephalpha": "0.1.0",
"python/frameworks/agentspec": "0.1.0",
"python/frameworks/promptflow": "0.1.0",
"go/langchaingo": "0.1.0",
"ruby": "0.1.0",
"ruby/roast": "0.1.0",
Expand Down
1 change: 1 addition & 0 deletions docs/python/auto-instrumentation.md
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ Each integration reports one of these statuses:
| `writer` | `writerai` | `traceAI-writer` | Automatic |
| `alephalpha` | `aleph_alpha_client` | `traceAI-alephalpha` | Automatic |
| `agentspec` | `pyagentspec` | `traceAI-agentspec` | Automatic |
| `promptflow` | `promptflow` | `traceAI-promptflow` | Automatic |
| `portkey` | `portkey-ai` | `traceAI-portkey` | Automatic |
| `watsonx` | `ibm_watsonx_ai` | `traceAI-watsonx` | Automatic |
| `replicate` | `replicate` | `traceAI-replicate` | Automatic |
Expand Down
1 change: 1 addition & 0 deletions python/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,7 @@ Each package below links to its own installation instructions, supported SDK ver
| `traceAI-pinecone` | [pinecone](frameworks/pinecone/) |
| `traceAI-pipecat` | [pipecat](frameworks/pipecat/) |
| `traceAI-portkey` | [portkey](frameworks/portkey/) |
| `traceAI-promptflow` | [promptflow](frameworks/promptflow/) |
| `traceai-pydantic-ai` | [pydantic-ai](frameworks/pydantic-ai/) |
| `traceAI-qdrant` | [qdrant](frameworks/qdrant/) |
| `traceAI-redis` | [redis-vector](frameworks/redis-vector/) |
Expand Down
1 change: 1 addition & 0 deletions python/fi_instrumentation/auto/_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ def _covered_by_openai(provider: str) -> str:
Integration("agentspec", ("pyagentspec",), "traceAI-agentspec", "traceai_agentspec", "AgentSpecInstrumentor"),
Integration("writer", ("writerai",), "traceAI-writer", "traceai_writer", "WriterInstrumentor"),
Integration("alephalpha", ("aleph_alpha_client",), "traceAI-alephalpha", "traceai_alephalpha", "AlephAlphaInstrumentor"),
Integration("promptflow", ("promptflow",), "traceAI-promptflow", "traceai_promptflow", "PromptflowInstrumentor"),
Integration("portkey", ("portkey_ai",), "traceAI-portkey", "traceai_portkey", "PortkeyInstrumentor", category="gateway"),
Integration("watsonx", ("ibm_watsonx_ai",), "traceAI-watsonx", "traceai_watsonx", "WatsonxInstrumentor"),
Integration("replicate", ("replicate",), "traceAI-replicate", "traceai_replicate", "ReplicateInstrumentor"),
Expand Down
5 changes: 5 additions & 0 deletions python/frameworks/promptflow/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Changelog

## [0.1.0] - pending release

- Instrument Microsoft Prompt flow (`promptflow-tracing`, `promptflow-core`): flex and DAG flow runs as `CHAIN` spans, nodes and `@trace` tools as child spans, LLM calls made through Prompt flow's OpenAI integration as `LLM` spans with model and token usage; sync, async and streaming.
60 changes: 60 additions & 0 deletions python/frameworks/promptflow/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
# traceAI-promptflow

OpenTelemetry instrumentation for [Microsoft Prompt flow](https://github.com/microsoft/promptflow)
(`promptflow-tracing`, `promptflow-core`; import `promptflow`).

```bash
pip install traceAI-promptflow
```

```python
from fi_instrumentation import register
from traceai_promptflow import PromptflowInstrumentor

PromptflowInstrumentor().instrument(tracer_provider=register(project_name="my-project"))

# then run flows or call @trace functions as usual
```

With `fi-instrumentation-otel` auto-instrumentation the `promptflow` integration is enabled whenever
`promptflow` is installed.

Prompt flow ships with tracing off (`PF_DISABLE_TRACING` defaults to `true`). `instrument()` turns
it on unless the variable is already set, and `uninstrument()` puts it back. Setting
`PF_DISABLE_TRACING=true` yourself keeps Prompt flow tracing disabled.

## What is traced

Prompt flow already opens a span for each `@trace` function, DAG node or tool and flow run (flex
and DAG). This instrumentor routes those spans through traceAI instead of the global tracer
provider:

* the flow run, or an outermost `@trace` function, is a `CHAIN` span; nodes and nested `@trace`
functions are child `TOOL` spans (`gen_ai.tool.name` is the function name). Calls made through
Prompt flow's OpenAI integration are `LLM` spans with the response model and token usage;
embedding vectors are never exported. `gen_ai.provider.name` is `promptflow` on every span;
* sync, async and generator results; a generator span ends on exhaustion, error, `close()` or
when the generator is dropped. An early end is marked `promptflow.stream.incomplete`;
* errors are re-raised unchanged and recorded as `ERROR` with `error.type` (the class name); the
message and traceback are never exported;
* inputs and outputs are exported as `input.value` / `output.value` and follow the `TraceConfig`
hide settings. Prompt flow's own `inputs`, `output`, `prompt.template`, `prompt.variables` and
`promptflow.*` events are never exported, whichever Prompt flow code path writes them (for
example a Prompty call): the span drops them, so hiding works for them too. If an installed
Prompt flow version lacks a content hook this package patches, one warning is logged and the
content stays masked; if it lacks `start_as_current_span`, nothing is instrumented;
* LLM and embedding spans carry `gen_ai.request.model` and the request parameters (temperature,
top_p, max tokens, penalties, seed, stop) when the call's inputs include them;
* a failure inside the instrumentation never reaches your call.

`PF_DISABLE_TRACING=false` is set process-wide, so child processes inherit it, but the patches are
not: a worker process has to call `instrument()` itself.

Limits: the patches live in the instrumenting process. Prompt flow batch runs that execute lines in
worker processes, and the serving `/feedback` endpoint (which opens its own span on the global
tracer provider), are not covered, so their spans do not follow the `TraceConfig` hide rules.

Do not also instrument the OpenAI SDK for the calls Prompt flow already traces: you would get a
second LLM span for each.

Supported SDK range: `promptflow-tracing >= 1.12.0, < 2`.
29 changes: 29 additions & 0 deletions python/frameworks/promptflow/pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
[tool.poetry]
name = "traceAI-promptflow"
version = "0.1.0"
description = "OpenTelemetry instrumentation for Microsoft Prompt flow (promptflow-tracing)"
authors = ["Future AGI <no-reply@futureagi.com>"]
readme = "README.md"
license = "Apache-2.0"
packages = [{ include = "traceai_promptflow" }]
classifiers = ["Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Framework :: OpenTelemetry :: Instrumentations"]

[tool.poetry.dependencies]
python = ">=3.10,<3.14"
fi-instrumentation-otel = ">=1.1.0"
promptflow-tracing = ">=1.12.0,<2"
wrapt = ">=1.15.0"

[tool.poetry.group.dev.dependencies]
pytest = ">=7"
pytest-asyncio = ">=0.23"
promptflow-core = ">=1.12.0,<2"
openai = ">=1.0"
httpx = ">=0.23"

[tool.poetry.urls]
Repository = "https://github.com/future-agi/traceAI/tree/dev/python/frameworks/promptflow"

[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
192 changes: 192 additions & 0 deletions python/frameworks/promptflow/tests/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,192 @@
"""Offline Prompt flow harness: real promptflow-tracing / promptflow-core, in-memory exporter."""
import json
import os
import textwrap

import pytest
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter

SECRET_INPUT = "synthetic-input-DO-NOT-EXPORT"
SECRET_ERROR = "synthetic-error-message-DO-NOT-EXPORT"


def attrs(span):
return dict(span.attributes)


def blob(span):
"""Every attribute, event name and event attribute of a span, as one string."""
events = [(e.name, dict(e.attributes or {})) for e in span.events]
return json.dumps([attrs(span), events, span.status.description], default=str)


def by_name(spans, name):
found = [s for s in spans if s.name == name or s.name.endswith("." + name)]
assert len(found) == 1, (name, [s.name for s in spans])
return found[0]


def write_dag_flow(root, fail=False):
"""A two-node DAG flow: greet -> shout. ``fail`` makes ``shout`` raise."""
(root / "flow.dag.yaml").write_text(
textwrap.dedent(
"""\
inputs:
question:
type: string
outputs:
answer:
type: string
reference: ${shout.output}
nodes:
- name: greet
type: python
source:
type: code
path: greet.py
inputs:
question: ${inputs.question}
- name: shout
type: python
source:
type: code
path: shout.py
inputs:
text: ${greet.output}
"""
)
)
(root / "greet.py").write_text(
"from promptflow.core import tool\n\n@tool\ndef greet(question: str) -> str:\n return 'hello ' + question\n"
)
body = f"raise ValueError({SECRET_ERROR!r})" if fail else "return text.upper()"
(root / "shout.py").write_text(
f"from promptflow.core import tool\n\n@tool\ndef shout(text: str) -> str:\n {body}\n"
)
return str(root / "flow.dag.yaml")


def write_flex_flow(root):
"""A flex flow whose entry calls one ``@trace`` helper."""
(root / "flow.flex.yaml").write_text("entry: flexapp:answer\n")
(root / "flexapp.py").write_text(
"from promptflow.tracing import trace\n\n"
"@trace\ndef helper(x):\n return x * 2\n\n"
"def answer(n: int) -> int:\n return helper(n) + 1\n"
)
return str(root / "flow.flex.yaml")


@pytest.fixture
def provider():
exporter = InMemorySpanExporter()
tracer_provider = TracerProvider()
tracer_provider.add_span_processor(SimpleSpanProcessor(exporter))
tracer_provider.exporter = exporter
yield tracer_provider
tracer_provider.shutdown()


@pytest.fixture(autouse=True)
def _tracing_env(monkeypatch):
# promptflow ships with tracing off; the instrumentor must switch it on by itself.
monkeypatch.delenv("PF_DISABLE_TRACING", raising=False)


@pytest.fixture
def instrument(provider):
from traceai_promptflow import PromptflowInstrumentor

instrumentors = []

def start(**kwargs):
instrumentor = PromptflowInstrumentor()
instrumentor.instrument(tracer_provider=provider, **kwargs)
instrumentors.append(instrumentor)
return provider.exporter

yield start
for instrumentor in reversed(instrumentors):
instrumentor.uninstrument()


@pytest.fixture
def spans(instrument):
return instrument()


class _OpenAIStub:
"""A loopback chat-completions endpoint, for the Prompty client that builds its own HTTP client."""

def __init__(self):
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer

stub = self
self.requests = []

class Handler(BaseHTTPRequestHandler):
def log_message(self, *args):
pass

def do_POST(self):
size = int(self.headers.get("Content-Length") or 0)
stub.requests.append(json.loads(self.rfile.read(size) or b"{}"))
body = json.dumps({
"id": "chatcmpl-1", "object": "chat.completion", "created": 1, "model": "gpt-served",
"choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "Tokyo"}}],
"usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5},
}).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)

self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
threading.Thread(target=self.server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True).start()
self.url = f"http://127.0.0.1:{self.server.server_address[1]}/v1"

def close(self):
self.server.shutdown()
self.server.server_close()


TEMPLATE_MARKER = "synthetic-template-DO-NOT-EXPORT"


@pytest.fixture
def prompty(tmp_path):
"""A real Prompty file whose OpenAI endpoint is a loopback stub (no external network)."""
stub = _OpenAIStub()
path = tmp_path / "ask.prompty"
path.write_text(
textwrap.dedent(
f"""\
---
name: ask
model:
api: chat
configuration:
type: openai
model: gpt-4o
base_url: {stub.url}
api_key: synthetic-openai-key-DO-NOT-EXPORT
parameters:
temperature: 0.2
max_tokens: 16
inputs:
question:
type: string
---
system:
{TEMPLATE_MARKER} answer briefly.
user:
{{{{question}}}}
"""
)
)
yield str(path)
stub.close()
Loading
Loading