|
69 | 69 | @inline Token(kind::TokenKinds.Kind, has_entities::Bool, raw::SubString) = |
70 | 70 | Token(kind, has_entities, raw.offset, raw.ncodeunits) |
71 | 71 |
|
72 | | -# Every span the scanner emits has both edges on UTF-8 character boundaries, so a view |
73 | | -# can be rebuilt straight from the stored (offset, ncodeunits) fields with no index |
74 | | -# walk. The invariant is structural: `NAME_BYTE_TABLE` classifies every byte 0x80–0xFF |
75 | | -# (the UTF-8 lead/continuation bytes) as a name byte, so name scans only stop on an |
76 | | -# ASCII byte or EOF, and every other span edge lands on an ASCII sentinel ('<', '>', |
77 | | -# '/', quotes, '-->', ']]>', '?>') or EOF. ASCII bytes are character boundaries by |
78 | | -# construction — even in malformed input. This is the package's one deliberate |
79 | | -# relaxation of the "never index ± 1 on strings" rule; see issue #109. |
80 | | -# |
81 | | -# `SubString`'s internal `Val(:noshift)` constructor takes the two fields directly |
82 | | -# (`base/strings/substring.jl:39-46` as of 1.12.6), unlike the checked constructor |
83 | | -# whose unconditional `nextind` re-derives the byte length even under `@inbounds`. |
84 | | -# The `@boundscheck` block preserves full validation under `--check-bounds=yes` (as in |
85 | | -# `Pkg.test`); callers wrap the call in `@inbounds`, so production builds elide it. |
86 | | -# Elision reaches one inlining level only, and only into callees that actually inline, |
87 | | -# so the constructor call below carries its own `@inbounds` plus a callsite `@inline`: |
88 | | -# Base's constructor validates under its own `@boundscheck` (a `prevind` plus two |
89 | | -# `isvalid` — `substring.jl:40-44`) and does not inline on its own, which would leave |
90 | | -# that validation running. The block above already covers it, and `--check-bounds=yes` |
91 | | -# re-enables every check regardless (#111). |
92 | | -# If Base ever drops the constructor, the checked index-walking fallback takes over. |
93 | | -if hasmethod(SubString{String}, Tuple{String, Int, Int, Val{:noshift}}) |
94 | | - @inline function _noshift_substring(s::T, offset::Int, ncu::Int) where {T <: AbstractString} |
95 | | - @boundscheck begin |
96 | | - n = ncodeunits(s) |
97 | | - (0 <= offset && 0 <= ncu && offset + ncu <= n) || |
98 | | - throw(BoundsError(s, (offset + 1):(offset + ncu))) |
99 | | - if ncu != 0 |
100 | | - isvalid(s, offset + 1) || throw(StringIndexError(s, offset + 1)) |
101 | | - offset + ncu == n || isvalid(s, offset + ncu + 1) || |
102 | | - throw(StringIndexError(s, offset + ncu + 1)) |
103 | | - end |
104 | | - end |
| 72 | +# Token views are rebuilt straight from the stored (offset, ncodeunits) fields — the |
| 73 | +# package's one deliberate relaxation of the "never index ± 1 on strings" rule (#109). |
| 74 | +# Safe because every span edge the scanner emits is an ASCII byte or EOF: |
| 75 | +# `NAME_BYTE_TABLE` classifies 0x80–0xFF as name bytes, so scans only stop on ASCII, |
| 76 | +# and ASCII bytes are UTF-8 character boundaries even in malformed input. |
| 77 | + |
| 78 | +# Checked reference reconstruction: index walks through the public constructor, which |
| 79 | +# validates both edges and yields the empty view natively (branchless on purpose — see |
| 80 | +# the union-return note below). |
| 81 | +@inline _checked_substring(s::AbstractString, offset::Int, ncu::Int) = |
| 82 | + SubString(s, offset + 1, prevind(s, offset + ncu + 1)) |
| 83 | + |
| 84 | +# One body per build, selected at load time (precompile caches are keyed by the flag): |
| 85 | +# `--check-bounds=yes` (as in `Pkg.test`) compiles the checked reconstruction wholesale |
| 86 | +# — an intra-body `@boundscheck` split would leave two construction paths and re-box |
| 87 | +# accessor chains' union returns (#113). Default builds compile the bare noshift store |
| 88 | +# (`substring.jl:39-46` @ 1.12.6); its call needs `@inbounds` plus a callsite |
| 89 | +# `@inline`, since elision reaches one inlining level and Base's self-validating |
| 90 | +# constructor does not inline on its own (#111). |
| 91 | +if hasmethod(SubString{String}, Tuple{String, Int, Int, Val{:noshift}}) && |
| 92 | + Base.JLOptions().check_bounds != 1 |
| 93 | + @inline _noshift_substring(s::T, offset::Int, ncu::Int) where {T <: AbstractString} = |
105 | 94 | @inbounds @inline SubString{T}(s, offset, ncu, Val(:noshift)) |
106 | | - end |
107 | 95 | else |
108 | | - @inline function _noshift_substring(s::AbstractString, offset::Int, ncu::Int) |
109 | | - ncu == 0 && return SubString(s, 1, 0) |
110 | | - SubString(s, offset + 1, prevind(s, offset + ncu + 1)) |
111 | | - end |
| 96 | + _noshift_substring(s::AbstractString, offset::Int, ncu::Int) = |
| 97 | + @inline _checked_substring(s, offset, ncu) |
112 | 98 | end |
113 | 99 |
|
114 | 100 | # Recover the token's text as a zero-copy `SubString` of its source `data` — a direct |
|
0 commit comments