[algo] fix: remove dead code in GPG advantage estimator

`compute_gpg_outcome_advantage` builds an `id2std` dict (and the per-group
std/constant assignments that populate it) that is never read: the
normalization on the final line divides by `f_norm`, not by the group std.
It also accepts an `alpha` parameter that is unconditionally overwritten by
`alpha = bsz / count_nonzero(scores)` before its first use, so the
caller-supplied value can never take effect. No caller passes `alpha` (the
advantage dispatch in ray_trainer.py builds kwargs without it).

Remove the unused `id2std` dict and the no-op `alpha` parameter. Behavior is
unchanged; the function still applies the documented N/N_nonzero scaling to
group-centered scores. Add a CPU unit test covering the scaling and the
singleton-group case.

Addresses issue #6478 (Item B). Item A (clamp) is handled separately in #6538.

Author: EazyReal

tests/trainer/ppo/test_gpg_advantage_on_cpu.py

@@ -0,0 +1,61 @@
+# Copyright 2025 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import torch
+
+from verl.trainer.ppo.core_algos import compute_gpg_outcome_advantage
+
+
+def test_gpg_singleton_group_returns_raw_score():
+    """A single response whose score is non-zero gets advantage == raw score.
+
+    For a singleton group the group mean is 0, and with every score non-zero
+    the GPG scale ``alpha = bsz / count_nonzero(scores)`` is 1, so the
+    advantage reduces to the raw (masked) reward.
+    """
+    token_level_rewards = torch.tensor([[0.0, 3.0, 0.0]], dtype=torch.float32)
+    response_mask = torch.tensor([[1.0, 1.0, 1.0]], dtype=torch.float32)
+    index = np.array(["prompt-a"], dtype=object)
+
+    advantages, returns = compute_gpg_outcome_advantage(
+        token_level_rewards=token_level_rewards,
+        response_mask=response_mask,
+        index=index,
+    )
+
+    raw_score = token_level_rewards.sum(dim=-1, keepdim=True) * response_mask
+    torch.testing.assert_close(advantages, raw_score)
+    torch.testing.assert_close(returns, advantages)
+
+
+def test_gpg_applies_n_over_nonzero_scaling():
+    """GPG scales centered scores by ``N / N_nonzero``.
+
+    With two responses in one group (scores 4 and 0) the group mean is 2.
+    ``alpha = bsz / count_nonzero = 2 / 1 = 2``, so each advantage is
+    ``2 * (score - 2)``.
+    """
+    token_level_rewards = torch.tensor([[4.0], [0.0]], dtype=torch.float32)
+    response_mask = torch.ones_like(token_level_rewards)
+    index = np.array(["prompt-a", "prompt-a"], dtype=object)
+
+    advantages, _ = compute_gpg_outcome_advantage(
+        token_level_rewards=token_level_rewards,
+        response_mask=response_mask,
+        index=index,
+    )
+
+    expected = torch.tensor([[4.0], [-4.0]], dtype=torch.float32)
+    torch.testing.assert_close(advantages, expected)

verl/trainer/ppo/core_algos.py

@@ -772,7 +772,6 @@ def compute_gpg_outcome_advantage(
     index: np.ndarray,
     epsilon: float = 1e-6,
     f_norm: float = 1.0,
-    alpha: float = 1.0,
     config=None,
     **kwargs,
 ):
@@ -788,7 +787,6 @@ def compute_gpg_outcome_advantage(
             shape: (bs,)
         epsilon: (float)
         f_norm: (float)
-        alpha: (float)
         config: (dict) algorithm config
 
     Returns:
@@ -801,7 +799,6 @@ def compute_gpg_outcome_advantage(
 
     id2score = defaultdict(list)
     id2mean = {}
-    id2std = {}
 
     with torch.no_grad():
         bsz = scores.shape[0]
@@ -814,11 +811,9 @@ def compute_gpg_outcome_advantage(
         for idx in id2score:
             if len(id2score[idx]) == 1:
                 id2mean[idx] = torch.tensor(0.0)
-                id2std[idx] = torch.tensor(1.0)
             elif len(id2score[idx]) > 1:
                 scores_tensor = torch.stack(id2score[idx])
                 id2mean[idx] = torch.mean(scores_tensor)
-                id2std[idx] = torch.std(scores_tensor)
             else:
                 raise ValueError(f"no score in prompt index: {idx}")
         for i in range(bsz):