From ae45422b11ec8ae63cceafd5377e4682440e56ab Mon Sep 17 00:00:00 2001 From: Gaurav Gandhi Date: Thu, 13 Aug 2026 12:59:33 +0530 Subject: [PATCH 1/2] fix(evaluation): record NOT_EVALUATED instead of dropping invocations with zero auto-rater samples LlmAsJudge.evaluate_invocations skipped straight to `continue` whenever an invocation's auto-rater call produced zero samples (e.g. the judge model's stream ended without emitting a response). The invocation was silently omitted from per_invocation_results entirely, shrinking the denominator downstream with no trace that anything went wrong. Same defect family as #6682 (NOT_EVALUATED metrics masked by a passing one) but one level up: here an invocation vanishes before it ever gets an eval_status. Append a PerInvocationResult defaulting to NOT_EVALUATED instead, consistent with how a genuinely-graded but missing metric is already represented elsewhere in this module. --- src/google/adk/evaluation/llm_as_judge.py | 10 +++ .../unittests/evaluation/test_llm_as_judge.py | 61 +++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/src/google/adk/evaluation/llm_as_judge.py b/src/google/adk/evaluation/llm_as_judge.py index c344490f7a..9101611376 100644 --- a/src/google/adk/evaluation/llm_as_judge.py +++ b/src/google/adk/evaluation/llm_as_judge.py @@ -191,6 +191,16 @@ async def evaluate_invocations( ) ) if not invocation_result_samples: + # The auto-rater produced no samples for this invocation (e.g. the + # judge model's stream ended without emitting a response). Record it + # as not evaluated instead of silently dropping it from the results, + # which would shrink the denominator downstream with no trace. + per_invocation_results.append( + PerInvocationResult( + actual_invocation=actual, + expected_invocation=expected, + ) + ) continue per_invocation_results.append( self.aggregate_per_invocation_samples(invocation_result_samples) diff --git a/tests/unittests/evaluation/test_llm_as_judge.py b/tests/unittests/evaluation/test_llm_as_judge.py index 61e0625958..f55a0d3209 100644 --- a/tests/unittests/evaluation/test_llm_as_judge.py +++ b/tests/unittests/evaluation/test_llm_as_judge.py @@ -289,3 +289,64 @@ async def test_evaluate_invocations_grades_criterion_only_metric( assert [r.eval_status for r in result.per_invocation_results] == [ EvalStatus.PASSED ] + + +@pytest.mark.asyncio +async def test_evaluate_invocations_records_not_evaluated_when_no_samples_produced( + mocker, +): + """An invocation whose auto-rater call produces zero samples (e.g. the judge + + model's stream ends without emitting a response) must still show up in + per_invocation_results as NOT_EVALUATED, not vanish from the results + entirely. Silently dropping it shrinks the denominator downstream with no + trace that anything went wrong. + """ + judge = PerInvocationReportingLlmAsJudge( + eval_metric=EvalMetric( + metric_name="test_metric", + criterion=LlmAsAJudgeCriterion( + threshold=0.5, + judge_model_options=JudgeModelOptions( + judge_model="gemini-2.5-flash", + judge_model_config=genai_types.GenerateContentConfig(), + num_samples=1, + ), + ), + ), + criterion_type=LlmAsAJudgeCriterion, + ) + + empty_judge_model = mocker.MagicMock() + + async def mock_generate_content_async_no_response(llm_request): + del llm_request + return + yield # pragma: no cover -- makes this an async generator. + + empty_judge_model.generate_content_async = ( + mock_generate_content_async_no_response + ) + judge._judge_model = empty_judge_model + + actual_invocations = [ + Invocation( + invocation_id="id1", + user_content=genai_types.Content( + parts=[genai_types.Part(text="user content 1")], + role="user", + ), + final_response=genai_types.Content( + parts=[genai_types.Part(text="final response 1")], + role="model", + ), + ) + ] + + result = await judge.evaluate_invocations(actual_invocations) + + assert len(result.per_invocation_results) == 1 + assert ( + result.per_invocation_results[0].eval_status == EvalStatus.NOT_EVALUATED + ) + assert result.per_invocation_results[0].score is None From 1e80b982710a8ffc5072c0cc43131350dfeb2d05 Mon Sep 17 00:00:00 2001 From: Gaurav Gandhi Date: Thu, 13 Aug 2026 20:36:52 +0530 Subject: [PATCH 2/2] fix(evaluation): spell out the NOT_EVALUATED row explicitly Matches hallucinations_v1's equivalent empty-row case (score=None, eval_status=EvalStatus.NOT_EVALUATED, rubric_scores=[]) instead of relying on PerInvocationResult's defaults, so both sites stay grep-matchable if those defaults ever move. --- src/google/adk/evaluation/llm_as_judge.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/google/adk/evaluation/llm_as_judge.py b/src/google/adk/evaluation/llm_as_judge.py index 9101611376..978a1ccb21 100644 --- a/src/google/adk/evaluation/llm_as_judge.py +++ b/src/google/adk/evaluation/llm_as_judge.py @@ -40,6 +40,7 @@ from .eval_metrics import RubricsBasedCriterion from .eval_metrics import RubricScore from .evaluator import _validate_invocation_lengths +from .evaluator import EvalStatus from .evaluator import EvaluationResult from .evaluator import Evaluator from .evaluator import PerInvocationResult @@ -195,10 +196,17 @@ async def evaluate_invocations( # judge model's stream ended without emitting a response). Record it # as not evaluated instead of silently dropping it from the results, # which would shrink the denominator downstream with no trace. + # Spelled out explicitly (matching hallucinations_v1's equivalent + # empty-row case) rather than relying on PerInvocationResult's + # defaults, so both sites stay grep-matchable if those defaults ever + # move. per_invocation_results.append( PerInvocationResult( actual_invocation=actual, expected_invocation=expected, + score=None, + eval_status=EvalStatus.NOT_EVALUATED, + rubric_scores=[], ) ) continue