Fix 2.9.0 regression: restore Cursor::match() remainder semantics (#1146)

* Restore remainder semantics in Cursor::match(), moving in-place matching to an internal method with native offset semantics

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* Add pathological cases covering attribute lists at and beyond the backtrack-limit boundary

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* Drop the inert m modifiers from BacktickParser's patterns

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Author: colinodell

CHANGELOG.md

@@ -13,6 +13,7 @@ Updates should follow the [Keep a CHANGELOG](https://keepachangelog.com/) princi
 - Optimized inline link destination parsing to scan the line in place, so its cost follows the length of the destination rather than the length of everything left in the block
 
 ### Fixed
+- Fixed a regression introduced in 2.9.0 where `Cursor::match()` treated text before the cursor as part of the match subject (#1145). Patterns were matched against the whole line at an offset, which silently changed the meaning of `\b`, `\B`, `\A`, lookbehinds, a `^` anywhere other than the very start of the pattern, and a leading `^` combined with the `m` modifier. `match()` once again matches against the remainder, exactly as it did in 2.8; the core parsers keep the optimized in-place matching via a new internal method with PCRE's native offset semantics, anchoring their patterns at the cursor with `\G`
 - Fixed heading permalinks rendered with `aria-hidden="true"` remaining in the keyboard tab order; they are now also given `tabindex="-1"`, as a focusable element removed from the accessibility tree has no accessible name to announce when focused (WCAG 4.1.2)
 - Fixed cloning a node breaking the link from the original node's children back to their parent, silently corrupting the document that node belonged to; detaching or inserting around those children afterwards could drop nodes from the tree
 - Fixed cloned nodes sharing their `data` with the node they were cloned from, so that setting an attribute on either one also set it on the other

src/Extension/Attributes/Util/AttributesHelper.php

@@ -24,7 +24,7 @@
 final class AttributesHelper
 {
     private const SINGLE_ATTRIBUTE = '\s*([.]-?[_a-z][^\s.}]*|[#][^\s}]+|' . RegexHelper::PARTIAL_ATTRIBUTENAME . RegexHelper::PARTIAL_ATTRIBUTEVALUESPEC . ')\s*';
-    private const ATTRIBUTE_LIST   = '/^{:?(' . self::SINGLE_ATTRIBUTE . ')+}/i';
+    private const ATTRIBUTE_LIST   = '/\G{:?(' . self::SINGLE_ATTRIBUTE . ')+}/i';
 
     /**
      * PCRE's `\s` matches the form feed that PHP's default trim charlist omits, so the
@@ -53,7 +53,7 @@ public static function parseAttributes(Cursor $cursor): array
         // matching individual attributes since they won't need to look ahead for the closing '}'
         // while dealing with the fact that attributes can technically contain curly braces.
         // So we'll just match the start and end braces up front.
-        $attributeExpression = $cursor->match(self::ATTRIBUTE_LIST);
+        $attributeExpression = $cursor->matchInPlace(self::ATTRIBUTE_LIST);
         if ($attributeExpression === null) {
             $cursor->restoreState($state);
 
@@ -66,7 +66,7 @@ public static function parseAttributes(Cursor $cursor): array
 
         /** @var array<string, mixed> $attributes */
         $attributes = [];
-        while ($attribute = \trim((string) $attributeCursor->match('/^' . self::SINGLE_ATTRIBUTE . '/i'), self::WHITESPACE)) {
+        while ($attribute = \trim((string) $attributeCursor->matchInPlace('/\G' . self::SINGLE_ATTRIBUTE . '/i'), self::WHITESPACE)) {
             if ($attribute[0] === '#') {
                 $attributes['id'] = \substr($attribute, 1);
 

src/Extension/CommonMark/Parser/Block/FencedCodeStartParser.php

@@ -27,7 +27,7 @@ public function tryStart(Cursor $cursor, MarkdownParserStateInterface $parserSta
         }
 
         $indent = $cursor->getIndent();
-        $fence  = $cursor->match('/^[ \t]*(?:`{3,}+(?!.*`)|~{3,})/');
+        $fence  = $cursor->matchInPlace('/\G[ \t]*(?:`{3,}+(?!.*`)|~{3,})/');
         if ($fence === null) {
             return BlockStart::none();
         }

src/Extension/CommonMark/Parser/Inline/BacktickParser.php

@@ -56,7 +56,7 @@ public function parse(InlineParserContext $inlineContext): bool
         if ($this->findMatchingTicks(\strlen($ticks), $cursor)) {
             $code = $cursor->getSubstring($currentPosition, $cursor->getPosition() - $currentPosition - \strlen($ticks));
 
-            $c = \preg_replace('/\n/m', ' ', $code) ?? '';
+            $c = \preg_replace('/\n/', ' ', $code) ?? '';
 
             if (
                 $c !== '' &&
@@ -110,7 +110,7 @@ private function findMatchingTicks(int $openTickLength, Cursor $cursor): bool
             return false;
         }
 
-        while ($ticks = $cursor->match('/`{1,' . self::MAX_BACKTICKS . '}/m')) {
+        while ($ticks = $cursor->matchInPlace('/`{1,' . self::MAX_BACKTICKS . '}/')) {
             $numTicks = \strlen($ticks);
 
             // Did we find the closer?

src/Extension/DescriptionList/Parser/DescriptionStartParser.php

@@ -29,7 +29,7 @@ public function tryStart(Cursor $cursor, MarkdownParserStateInterface $parserSta
         }
 
         $cursor->advanceToNextNonSpaceOrTab();
-        if ($cursor->match('/^:[ \t]+/') === null) {
+        if ($cursor->matchInPlace('/\G:[ \t]+/') === null) {
             return BlockStart::none();
         }
 

src/Extension/FrontMatter/FrontMatterParser.php

@@ -23,7 +23,7 @@ final class FrontMatterParser implements FrontMatterParserInterface
     /** @psalm-readonly */
     private FrontMatterDataParserInterface $frontMatterParser;
 
-    private const REGEX_FRONT_MATTER = '/^---\\R.*?\\R---\\R/s';
+    private const REGEX_FRONT_MATTER = '/\G---\\R.*?\\R---\\R/s';
 
     public function __construct(FrontMatterDataParserInterface $frontMatterParser)
     {
@@ -38,7 +38,7 @@ public function parse(string $markdownContent): MarkdownInputWithFrontMatter
         $cursor = new Cursor($markdownContent);
 
         // Locate the front matter
-        $frontMatter = $cursor->match(self::REGEX_FRONT_MATTER);
+        $frontMatter = $cursor->matchInPlace(self::REGEX_FRONT_MATTER);
         if ($frontMatter === null) {
             return new MarkdownInputWithFrontMatter($markdownContent);
         }
@@ -53,7 +53,7 @@ public function parse(string $markdownContent): MarkdownInputWithFrontMatter
         $data = $this->frontMatterParser->parse($frontMatter);
 
         // Advance through any remaining newlines which separated the front matter from the Markdown text
-        $trailingNewlines = $cursor->match('/^\R+/');
+        $trailingNewlines = $cursor->matchInPlace('/\G\R+/');
 
         // Calculate how many lines the Markdown is offset from the front matter by counting the number of newlines
         // Don't forget to add 1 because we stripped one out when trimming the trailing delims

src/Extension/Table/TableStartParser.php

@@ -107,7 +107,7 @@ private static function parseSeparator(Cursor $cursor): array
                         $cursor->advanceBy(1);
                     }
 
-                    if ($cursor->match('/^-+/') === null) {
+                    if ($cursor->matchInPlace('/\G-+/') === null) {
                         // Need at least one dash
                         return [];
                     }

src/Extension/TableOfContents/TableOfContentsPlaceholderParser.php

@@ -58,7 +58,7 @@ public function tryStart(Cursor $cursor, MarkdownParserStateInterface $parserSta
                 }
 
                 // The placeholder must be the only thing on the line
-                if ($cursor->match('/^' . \preg_quote($placeholder, '/') . '$/') === null) {
+                if ($cursor->matchInPlace('/\G' . \preg_quote($placeholder, '/') . '$/') === null) {
                     return BlockStart::none();
                 }
 

src/Parser/Cursor.php

@@ -538,94 +538,107 @@ public function isAtEnd(): bool
     }
 
     /**
-     * Try to match a regular expression
+     * Try to match a regular expression against the remainder of the line
+     *
+     * The subject begins at the cursor: text before the cursor is invisible to the pattern,
+     * so "^" and "\A" anchor at the cursor, and constructs which examine what precedes the
+     * match position (lookbehinds, "\b", "\B") see the start of a subject there rather than
+     * the characters actually preceding the cursor.
      *
      * Returns the matching text and advances to the end of that match
      *
      * @psalm-param non-empty-string $regex
      */
     public function match(string $regex): ?string
     {
-        // When a tab has been partially consumed the remainder is reconstructed with the
-        // leftover tab expanded into spaces, so matching must run against that reconstructed
-        // string rather than the raw line. This is rare; use the copy-based path to preserve
-        // the exact column arithmetic.
-        if ($this->partiallyConsumedTab) {
-            return $this->matchViaRemainder($regex);
-        }
-
-        // Match against the persistent line at the current byte offset instead of allocating a
-        // fresh copy of the remainder on every call. A leading "^" is rewritten to "\G" so the
-        // pattern still anchors to the cursor - a bare "^" only matches at the true start of the
-        // subject when a non-zero offset is supplied. Patterns that intentionally scan ahead
-        // (e.g. the backtick closer search) carry no leading "^" and are left untouched. This
-        // keeps repeated match() calls - such as that backtick scan - linear rather than O(n^2),
-        // since each call no longer copies the entire remaining line.
-        if ($regex[1] === '^') {
-            $regex = $regex[0] . '\\G' . \substr($regex, 2);
-        }
-
-        $bytePosition = $this->isMultibyte ? $this->byteOffset($this->currentPosition) : $this->currentPosition;
+        $subject = $this->getRemainder();
 
-        if (! \preg_match($regex, $this->line, $matches, \PREG_OFFSET_CAPTURE, $bytePosition)) {
+        if (! \preg_match($regex, $subject, $matches, \PREG_OFFSET_CAPTURE)) {
             return null;
         }
 
-        // $matches[0][0] contains the matched text; $matches[0][1] is its absolute byte offset in the line.
+        // $matches[0][0] contains the matched text; $matches[0][1] is its byte offset in the subject.
         if ($this->isMultibyte) {
-            // Convert the byte offset to a character advance relative to the cursor. The scanned gap
-            // is only the distance from the cursor to the match (zero for anchored patterns), never
-            // the whole line, so this stays linear across repeated calls.
-            $offset      = \mb_strlen(\substr($this->line, $bytePosition, $matches[0][1] - $bytePosition), 'UTF-8');
+            $offset      = \mb_strlen(\substr($subject, 0, $matches[0][1]), 'UTF-8');
             $matchLength = \mb_strlen($matches[0][0], 'UTF-8');
         } else {
-            $offset      = $matches[0][1] - $this->currentPosition;
+            $offset      = $matches[0][1];
             $matchLength = \strlen($matches[0][0]);
         }
 
-        $this->advanceBy($offset + $matchLength);
+        $advance = $offset + $matchLength;
+
+        // The remainder we matched against had any partially-consumed tab expanded into spaces,
+        // so those columns must be advanced by column instead of by character.
+        if ($this->partiallyConsumedTab) {
+            $charsToTab = 4 - ($this->column % 4);
+            if ($advance < $charsToTab) {
+                $this->advanceBy($advance, true);
+
+                return $matches[0][0];
+            }
+
+            $this->advanceBy($charsToTab, true);
+            $advance -= $charsToTab;
+        }
+
+        $this->advanceBy($advance);
 
         return $matches[0][0];
     }
 
     /**
-     * Slow path for match() used only when a tab has been partially consumed: match against a
-     * freshly-built remainder whose leftover tab is expanded into spaces, advancing by columns
-     * across that expansion. Kept separate so the common case avoids the remainder allocation.
+     * Try to match a regular expression at the cursor's position within the line, without
+     * copying the remainder
+     *
+     * Matches with PCRE's native offset semantics: the whole line is the subject, and matching
+     * starts at the cursor. "\G" anchors at the cursor; "^" anchors at the true start of the
+     * line (or after newlines under the "m" modifier); lookbehinds, "\b", and "\B" see the
+     * characters actually preceding the cursor. This differs from match(), whose subject begins
+     * at the cursor - a pattern written for match() migrates by replacing its leading "^" (or
+     * "\A") with "\G".
+     *
+     * Because no copy of the remainder is made, repeated calls stay linear: match() copies
+     * everything left in the line on every call, so scanning loops (such as the backtick closer
+     * search) would otherwise cost O(n^2).
+     *
+     * When a tab has been partially consumed, no position within the line can represent the
+     * cursor, so this falls back to matching the remainder with the leftover tab expanded into
+     * spaces; "\G" still anchors at the cursor there, but the line content before it is not
+     * visible in that case.
+     *
+     * @internal Planned to become public API in 2.10; until then the name and contract may
+     *           change without notice.
      *
      * @psalm-param non-empty-string $regex
      */
-    private function matchViaRemainder(string $regex): ?string
+    public function matchInPlace(string $regex): ?string
     {
-        $subject = $this->getRemainder();
+        // A partially-consumed tab means the remainder differs from the underlying line (the
+        // leftover tab expands into spaces), so no byte offset can represent the cursor.
+        if ($this->partiallyConsumedTab) {
+            return $this->match($regex);
+        }
 
-        if (! \preg_match($regex, $subject, $matches, \PREG_OFFSET_CAPTURE)) {
+        $bytePosition = $this->isMultibyte ? $this->byteOffset($this->currentPosition) : $this->currentPosition;
+
+        if (! \preg_match($regex, $this->line, $matches, \PREG_OFFSET_CAPTURE, $bytePosition)) {
             return null;
         }
 
+        // $matches[0][0] contains the matched text; $matches[0][1] is its absolute byte offset in the line.
         if ($this->isMultibyte) {
-            $offset      = \mb_strlen(\substr($subject, 0, $matches[0][1]), 'UTF-8');
+            // Convert the byte offset to a character advance relative to the cursor. The scanned gap
+            // is only the distance from the cursor to the match (zero for anchored patterns), never
+            // the whole line, so this stays linear across repeated calls.
+            $offset      = \mb_strlen(\substr($this->line, $bytePosition, $matches[0][1] - $bytePosition), 'UTF-8');
             $matchLength = \mb_strlen($matches[0][0], 'UTF-8');
         } else {
-            $offset      = $matches[0][1];
+            $offset      = $matches[0][1] - $this->currentPosition;
             $matchLength = \strlen($matches[0][0]);
         }
 
-        $advance = $offset + $matchLength;
-
-        // The remainder we matched against had the partially-consumed tab expanded into spaces,
-        // so those columns must be advanced by column instead of by character.
-        $charsToTab = 4 - ($this->column % 4);
-        if ($advance < $charsToTab) {
-            $this->advanceBy($advance, true);
-
-            return $matches[0][0];
-        }
-
-        $this->advanceBy($charsToTab, true);
-        $advance -= $charsToTab;
-
-        $this->advanceBy($advance);
+        $this->advanceBy($offset + $matchLength);
 
         return $matches[0][0];
     }

src/Util/LinkParserHelper.php

@@ -46,7 +46,7 @@ public static function parseLinkDestination(Cursor $cursor): ?string
 
     public static function parseLinkLabel(Cursor $cursor): int
     {
-        $match = $cursor->match('/^\[(?:[^\\\\\[\]]|\\\\.){0,1000}\]/');
+        $match = $cursor->matchInPlace('/\G\[(?:[^\\\\\[\]]|\\\\.){0,1000}\]/');
         if ($match === null) {
             return 0;
         }
@@ -62,7 +62,7 @@ public static function parseLinkLabel(Cursor $cursor): int
 
     public static function parsePartialLinkLabel(Cursor $cursor): ?string
     {
-        return $cursor->match('/^(?:[^\\\\\[\]]++|\\\\.?)*+/');
+        return $cursor->matchInPlace('/\G(?:[^\\\\\[\]]++|\\\\.?)*+/');
     }
 
     /**
@@ -72,7 +72,7 @@ public static function parsePartialLinkLabel(Cursor $cursor): ?string
      */
     public static function parseLinkTitle(Cursor $cursor): ?string
     {
-        if ($title = $cursor->match('/' . RegexHelper::PARTIAL_LINK_TITLE . '/')) {
+        if ($title = $cursor->matchInPlace('/\G' . RegexHelper::PARTIAL_LINK_TITLE_UNANCHORED . '/')) {
             // Chop off quotes from title and unescape
             return RegexHelper::unescape(\substr($title, 1, -1));
         }
@@ -84,7 +84,7 @@ public static function parsePartialLinkTitle(Cursor $cursor, string $endDelimite
     {
         $endDelimiter = \preg_quote($endDelimiter, '/');
         $regex        = \sprintf('/(%s|[^%s\x00])*(?:%s)?/', RegexHelper::PARTIAL_ESCAPED_CHAR, $endDelimiter, $endDelimiter);
-        if (($partialTitle = $cursor->match($regex)) === null) {
+        if (($partialTitle = $cursor->matchInPlace($regex)) === null) {
             return null;
         }
 
@@ -156,7 +156,7 @@ private static function parseDestinationBraces(Cursor $cursor): ?string
             self::$lastCursor = \WeakReference::create($cursor);
         }
 
-        if ($res = $cursor->match(RegexHelper::REGEX_LINK_DESTINATION_BRACES)) {
+        if ($res = $cursor->matchInPlace('/\G' . RegexHelper::PARTIAL_LINK_DESTINATION_BRACES . '/')) {
             self::$lastCursorLacksClosingBrace = false;
 
             // Chop off surrounding <..>: