Fix TextChunker token-counted paragraph merge

Author: pragnyanramtha

dotnet/src/SemanticKernel.Core/Text/TextChunker.cs

@@ -202,9 +202,13 @@ private static List<string> ProcessParagraphs(List<string> paragraphs, int adjus
                 {
                     var newSecondLastParagraph = string.Join(" ", secondLastParagraphTokens);
                     var newLastParagraph = string.Join(" ", lastParagraphTokens);
+                    var mergedParagraph = $"{newSecondLastParagraph} {newLastParagraph}";
 
-                    paragraphs[paragraphs.Count - 2] = $"{newSecondLastParagraph} {newLastParagraph}";
-                    paragraphs.RemoveAt(paragraphs.Count - 1);
+                    if (GetTokenCount(mergedParagraph, tokenCounter) <= adjustedMaxTokensPerParagraph)
+                    {
+                        paragraphs[paragraphs.Count - 2] = mergedParagraph;
+                        paragraphs.RemoveAt(paragraphs.Count - 1);
+                    }
                 }
             }
         }

dotnet/src/SemanticKernel.UnitTests/Text/TextChunkerTests.cs

@@ -558,6 +558,45 @@ public void CanSplitTextParagraphsWithCustomTokenCounter()
         Assert.Equal(expected, result);
     }
 
+    [Fact]
+    public void SplitPlainTextParagraphsWithCustomTokenCounterDoesNotMergePastTokenLimit()
+    {
+        List<string> input =
+        [
+            "abcdefghijklmnopqr",
+            "abc"
+        ];
+
+        var expected = new[]
+        {
+            "abcdefghijklmnopqr",
+            "abc"
+        };
+
+        var result = TextChunker.SplitPlainTextParagraphs(input, 20, tokenCounter: static (input) => input.Length);
+
+        Assert.Equal(expected, result);
+    }
+
+    [Fact]
+    public void SplitPlainTextParagraphsWithCustomTokenCounterStillMergesWhenCandidateFits()
+    {
+        List<string> input =
+        [
+            "abcdefghijklmnop",
+            "abc"
+        ];
+
+        var expected = new[]
+        {
+            "abcdefghijklmnop abc"
+        };
+
+        var result = TextChunker.SplitPlainTextParagraphs(input, 20, tokenCounter: static (input) => input.Length);
+
+        Assert.Equal(expected, result);
+    }
+
     [Fact]
     public void CanSplitTextParagraphsWithOverlapAndCustomTokenCounter()
     {