Skip to content

Commit 37cb0f1

Browse files
TeodorEscuUiPathclaude
TeodorEscuUiPath
and
claude
committed
fix(eval): fall back AgentRunHistory to final output when no tool was called
trace_to_str() only ever renders spans carrying tool.name, so a run where the agent answered in plain text and made zero tool calls produced an empty AgentRunHistory - not because nothing happened, but because trace_to_str had nothing tool-shaped to render. This hid the agent's real final answer from both trajectory evaluators, matching UV-16309's "AgentRunHistory omits the agent's own text responses, causing false 0s (empty) or unearned high scores (non-empty)" description. Both LegacyTrajectoryEvaluator and LLMJudgeTrajectoryEvaluator (via BaseLLMTrajectoryEvaluator) now append WorkloadExecution.workload_output - the agent's actual final answer, independent of the trace - whenever it's non-empty, so the judge always sees what the agent answered. Co-Authored-By: Claude Sonnet 5 <[email protected]>
1 parent 1bdff0c commit 37cb0f1

2 files changed

Lines changed: 33 additions & 2 deletions

File tree

‎packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ async def evaluate(
101101
evaluation_prompt = self._create_evaluation_prompt(
102102
expected_agent_behavior=workload_execution.expected_agent_behavior,
103103
agent_run_history=workload_execution.workload_trace,
104+
workload_output=workload_execution.workload_output,
104105
)
105106
llm_response = await self._get_llm_response(evaluation_prompt)
106107

@@ -113,6 +114,7 @@ def _create_evaluation_prompt(
113114
self,
114115
expected_agent_behavior: Any,
115116
agent_run_history: Any,
117+
workload_output: Any = None,
116118
) -> str:
117119
"""Create the evaluation prompt for the LLM."""
118120
# Validate that expected agent behavior is not empty
@@ -143,6 +145,18 @@ def _create_evaluation_prompt(
143145
else:
144146
agent_run_history = str(agent_run_history)
145147

148+
# trace_to_str only ever renders tool-call spans, so a tool-free
149+
# text-only response leaves agent_run_history empty even though the
150+
# agent genuinely answered. Append the actual final output so the
151+
# judge always sees what the agent answered (UV-16309).
152+
if workload_output:
153+
final_output_section = f"Agent Final Response:\n{workload_output}"
154+
agent_run_history = (
155+
f"{agent_run_history}\n\n{final_output_section}"
156+
if agent_run_history
157+
else final_output_section
158+
)
159+
146160
formatted_prompt = formatted_prompt.replace(
147161
self.agent_run_history_placeholder,
148162
agent_run_history,

‎packages/uipath/src/uipath/eval/evaluators/llm_judge_trajectory_evaluator.py‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,25 @@ async def evaluate(
7979
return await super().evaluate(workload_execution, evaluation_criteria)
8080

8181
def _get_actual_output(self, workload_execution: WorkloadExecution) -> Any:
82-
"""Get the actual output from the workload execution."""
83-
return trace_to_str(workload_execution.workload_trace)
82+
"""Get the actual output from the workload execution.
83+
84+
`trace_to_str` only ever renders tool-call spans, so a run where the
85+
agent responded with plain text and made no tool calls at all
86+
produces an empty string here - not because nothing happened, but
87+
because there was nothing for `trace_to_str` to render. Append the
88+
agent's actual final output so the judge always sees what the agent
89+
answered, not just which tools it used (UV-16309).
90+
"""
91+
history = trace_to_str(workload_execution.workload_trace)
92+
final_output = workload_execution.workload_output
93+
if final_output:
94+
final_output_section = f"Agent Final Response:\n{final_output}"
95+
history = (
96+
f"{history}\n\n{final_output_section}"
97+
if history
98+
else final_output_section
99+
)
100+
return history
84101

85102
def _get_expected_output(
86103
self, evaluation_criteria: TrajectoryEvaluationCriteria

0 commit comments

Comments
 (0)