Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Appearance settings
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion 2 pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "uipath"
version = "2.1.76"
version = "2.1.77"
description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
readme = { file = "README.md", content-type = "text/markdown" }
requires-python = ">=3.10"
Expand Down
115 changes: 0 additions & 115 deletions 115 src/uipath/_cli/_evals/_models/_trajectory_span.py

This file was deleted.

8 changes: 6 additions & 2 deletions 8 src/uipath/eval/evaluators/trajectory_evaluator.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,16 @@
from opentelemetry.sdk.trace import ReadableSpan
from pydantic import field_validator

from uipath._cli._evals._models._trajectory_span import TrajectoryEvaluationTrace
from uipath.eval.models import EvaluationResult

from ..._services import UiPathLlmChatService
from ..._utils.constants import COMMUNITY_agents_SUFFIX
from ..models.models import AgentExecution, LLMResponse, NumericEvaluationResult
from ..models.models import (
AgentExecution,
LLMResponse,
NumericEvaluationResult,
TrajectoryEvaluationTrace,
)
from .base_evaluator import BaseEvaluator


Expand Down
111 changes: 110 additions & 1 deletion 111 src/uipath/eval/models/models.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
"""Models for evaluation framework including execution data and evaluation results."""

from dataclasses import dataclass
from enum import IntEnum
from typing import Annotated, Any, Dict, Literal, Optional, Union
from typing import Annotated, Any, Dict, List, Literal, Optional, Union

from opentelemetry.sdk.trace import ReadableSpan
from pydantic import BaseModel, ConfigDict, Field
Expand Down Expand Up @@ -113,3 +114,111 @@ def from_int(cls, value):
return cls(value)
else:
raise ValueError(f"{value} is not a valid EvaluatorType value")


@dataclass
class TrajectoryEvaluationSpan:
"""Simplified span representation for trajectory evaluation.

Contains span information needed for evaluating agent execution paths,
excluding timestamps which are not useful for trajectory analysis.
"""

name: str
status: str
attributes: Dict[str, Any]
parent_name: Optional[str] = None
events: Optional[List[Dict[str, Any]]] = None

def __post_init__(self):
"""Initialize default values."""
if self.events is None:
self.events = []

@classmethod
def from_readable_span(
cls, span: ReadableSpan, parent_spans: Optional[Dict[int, str]] = None
) -> "TrajectoryEvaluationSpan":
"""Convert a ReadableSpan to a TrajectoryEvaluationSpan.

Args:
span: The OpenTelemetry ReadableSpan to convert
parent_spans: Optional mapping of span IDs to names for parent lookup

Returns:
TrajectoryEvaluationSpan with relevant data extracted
"""
# Extract status
status_map = {0: "unset", 1: "ok", 2: "error"}
status = status_map.get(span.status.status_code.value, "unknown")

# Extract attributes - keep all attributes for now
attributes = {}
if span.attributes:
attributes = dict(span.attributes)

# Get parent name if available
parent_name = None
if span.parent and parent_spans and span.parent.span_id in parent_spans:
parent_name = parent_spans[span.parent.span_id]

# Extract events (without timestamps)
events = []
if hasattr(span, "events") and span.events:
for event in span.events:
event_data = {
"name": event.name,
"attributes": dict(event.attributes) if event.attributes else {},
}
events.append(event_data)

return cls(
name=span.name,
status=status,
attributes=attributes,
parent_name=parent_name,
events=events,
)

def to_dict(self) -> Dict[str, Any]:
"""Convert to dictionary for JSON serialization."""
return {
"name": self.name,
"status": self.status,
"parent_name": self.parent_name,
"attributes": self.attributes,
"events": self.events,
}


class TrajectoryEvaluationTrace(BaseModel):
"""Container for a collection of trajectory evaluation spans."""

spans: List[TrajectoryEvaluationSpan]

@classmethod
def from_readable_spans(
cls, spans: List[ReadableSpan]
) -> "TrajectoryEvaluationTrace":
"""Convert a list of ReadableSpans to TrajectoryEvaluationTrace.

Args:
spans: List of OpenTelemetry ReadableSpans to convert

Returns:
TrajectoryEvaluationTrace with converted spans
"""
# Create a mapping of span IDs to names for parent lookup
span_id_to_name = {span.get_span_context().span_id: span.name for span in spans}

evaluation_spans = [
TrajectoryEvaluationSpan.from_readable_span(span, span_id_to_name)
for span in spans
]

return cls(spans=evaluation_spans)

class Config:
"""Pydantic configuration."""

arbitrary_types_allowed = True
Morty Proxy This is a proxified and sanitized view of the page, visit original site.