fix(floats): disambiguate decimal delimiter

Treat a shared decimal and delimiter byte as a delimiter outside quoted fields and as a decimal inside quoted fields.

Carry the active delimiter context into numeric parsing. Preserve custom decimal behavior for single-value parsing, BigFloat, and Number across string, byte-buffer, view, and IO sources.

Fixes #185

Author: quinnj

src/Parsers.jl

@@ -101,7 +101,7 @@ end
   * `closequotechar='"'`: the ascii character that signals the end of a quoted field
   * `escapechar='"'`: an ascii character used to "escape" a `closequotechar` within a quoted field
   * `delim=','`: if `nothing`, no delimiter will be checked for; if a `Char` or `String`, a delimiter will be checked for directly after parsing a value or `closequotechar`; a newline (`\\n`), return (`\\r`), or CRLF (`"\\r\\n"`) are always considered "delimiters", in addition to EOF
-  * `decimal='.'`: an ascii character to be used when parsing float values that separates a decimal value
+  * `decimal='.'`: an ascii character to be used when parsing float values that separates a decimal value. When `decimal` matches `delim`, the character is treated as a delimiter outside quoted values and as a decimal inside quoted values.
   * `trues=nothing`: if `nothing`, `Bool` parsing will only check for the string `true` or an `Integer` value of `1` as valid values for `true`; as a `Vector{String}`, each string value will be checked to indicate a valid `true` value
   * `falses=nothing`: if `nothing`, `Bool` parsing will only check for the string `false` or an `Integer` value of `0` as valid values for `false`; as a `Vector{String}`, each string value will be checked to indicate a valid `false` value
   * `dateformat=nothing`: if `nothing`, `Date`, `DateTime`, and `Time` parsing will use a default `Dates.DateFormat` object while parsing; a `String` or `Dates.DateFormat` object can be provided for custom format parsing
@@ -437,7 +437,7 @@ end
 
 # condensed version of xparse that doesn't worry about quoting or delimiters; called from Parsers.parse/Parsers.tryparse
 _xparse2(conf::AbstractConf{T}, source::Union{AbstractVector{UInt8}, IO}, pos, len, opts::Options=OPTIONS, ::Type{S}=returntype(T)) where {T, S} =
-    Result(whitespace(false, false, false, true)(typeparser(opts)))(conf, source, pos, len, S)
+    Result(whitespace(false, false, false, true)(typeparser(opts, false)))(conf, source, pos, len, S)
 
 xparse2(::Type{T}, source::SourceType, pos, len, options=OPTIONS, ::Type{S}=returntype(T)) where {T, S} =
     result(T, xparse2(conf(T, options), source, pos, len, options, S))
@@ -493,6 +493,13 @@ function _has_groupmark(opts::Options, code::ReturnCode)
     return false
 end
 
+@inline function _effective_decimal(opts::Options, code::ReturnCode, checkdelim::Bool)
+    if checkdelim && !quoted(code) && opts.decimal == opts.delim
+        return 0xff
+    end
+    return opts.decimal
+end
+
 
 if isdefined(Base, :OncePerTask)
     const _get_bigint = OncePerTask{BigInt}(() -> BigInt(; nbits=256))

src/components.jl

@@ -376,13 +376,18 @@ function delimiter(checkdelim, delim, ignorerepeated, cmt, ignoreemptylines, str
     end
 end
 
-function typeparser(opts::Options)
+typeparser(opts::Options) = typeparser(opts, opts.flags.checkdelim)
+
+function typeparser(opts::Options, checkdelim::Bool)
     function(conf::AbstractConf{T}, source, pos, len, b, code, pl) where {T}
         Base.@_inline_meta
-        return typeparser(conf, source, pos, len, b, code, pl, opts)
+        return typeparser(conf, source, pos, len, b, code, pl, opts, checkdelim)
     end
 end
 
+typeparser(conf, source, pos, len, b, code, pl, opts, ::Bool) =
+    typeparser(conf, source, pos, len, b, code, pl, opts)
+
 # backwards compat
 function typeparser(conf, source, pos, len, b, code, opts::Options)
     pos, code, pl, x = typeparser(conf, source, pos, len, b, code, poslen(pos, 0), opts)
@@ -395,4 +400,4 @@ function typeparser(::Type{T}, source, pos, len, b, code, opts::Options) where {
 end
 
 typeparser(::Type{T}, source, pos, len, b, code, pl) where {T} =
-    typeparser(DefaultConf{T}(), source, pos, len, b, code, pl, Options())
+    typeparser(DefaultConf{T}(), source, pos, len, b, code, pl, Options())

src/floats.jl

@@ -62,22 +62,31 @@ end
     return _contiguous(source) && len < length(source) && @inbounds(source[len + 1]) == 0x00
 end
 
-@inline function _ends_on_delimiter(source::AbstractVector{UInt8}, len, options)
+@inline function _ends_on_delimiter(source::AbstractVector{UInt8}, len, options, checkdelim)
     0 < len <= length(source) || return false
     @inbounds b = source[len]
-    return b == UInt8('\n') || b == UInt8('\r') || (options.flags.checkdelim && options.delim == b)
+    return b == UInt8('\n') || b == UInt8('\r') || (checkdelim && options.delim == b)
 end
 
-function _bigfloat_buffer(source::AbstractVector{UInt8}, pos, len, b, code, options)
-    _, _, pl, _ = typeparser(DefaultConf{String}(), source, pos, len, b, code, poslen(pos, 0), options)
+function _bigfloat_poslen(source, pos, len, b, code, options, checkdelim)
+    if quoted(code) || checkdelim
+        _, _, pl, _ = typeparser(DefaultConf{String}(), source, pos, len, b, code, poslen(pos, 0), options)
+    else
+        _, _, pl, _ = findeof(source, pos, len, b, code, poslen(pos, 0), options)
+    end
+    return pl
+end
+
+function _bigfloat_buffer(source::AbstractVector{UInt8}, pos, len, b, code, options, checkdelim)
+    pl = _bigfloat_poslen(source, pos, len, b, code, options, checkdelim)
     return _copy_nulterminated(source, pl.pos, pl.len)
 end
 
-_bigfloat_buffer(source::AbstractString, pos, len, b, code, options) =
-    _bigfloat_buffer(codeunits(source), pos, len, b, code, options)
+_bigfloat_buffer(source::AbstractString, pos, len, b, code, options, checkdelim) =
+    _bigfloat_buffer(codeunits(source), pos, len, b, code, options, checkdelim)
 
-function _bigfloat_buffer(source::IO, pos, len, b, code, options)
-    _, _, pl, _ = typeparser(DefaultConf{String}(), source, pos, len, b, code, poslen(pos, 0), options)
+function _bigfloat_buffer(source::IO, pos, len, b, code, options, checkdelim)
+    pl = _bigfloat_poslen(source, pos, len, b, code, options, checkdelim)
     vlen = max(0, pl.len)
     buf = Vector{UInt8}(undef, vlen + 1)
     fastseek!(source, pl.pos - 1)
@@ -87,6 +96,13 @@ function _bigfloat_buffer(source::IO, pos, len, b, code, options)
     return buf
 end
 
+function _normalize_decimal!(buf, decimal)
+    @inbounds for i = 1:(length(buf) - 1)
+        buf[i] == decimal && (buf[i] = UInt8('.'))
+    end
+    return buf
+end
+
 @inline function _mpfr_strtofr(z, ptr, endptr, base, rounding)
     return ccall((:mpfr_strtofr, :libmpfr), Int32, (Ref{BigFloat}, Cstring, Ref{Ptr{UInt8}}, Int32, Base.MPFR.MPFRRoundingMode), z, ptr, endptr, base, rounding)
 end
@@ -100,19 +116,30 @@ end
     return pos, code, PosLen(pl.pos, max(0, pos - pl.pos)), z
 end
 
-function typeparser(::AbstractConf{BigFloat}, source, pos, len, b, code, pl, options)
+function typeparser(conf::AbstractConf{BigFloat}, source, pos, len, b, code, pl, options)
+    return typeparser(conf, source, pos, len, b, code, pl, options, options.flags.checkdelim)
+end
+
+function typeparser(::AbstractConf{BigFloat}, source, pos, len, b, code, pl, options, checkdelim::Bool)
     base = 0
     rounding = Base.MPFR.ROUNDING_MODE[]
     z = BigFloat(precision=Base.MPFR.DEFAULT_PRECISION[])
-    if _nulterminated(source, len)
+    decimal = _effective_decimal(options, code, checkdelim)
+    if decimal != UInt8('.')
+        buf = _bigfloat_buffer(source, pos, len, b, code, options, checkdelim)
+        decimal != 0xff && _normalize_decimal!(buf, decimal)
+        GC.@preserve buf begin
+            return _finish_bigfloat(source, pos, code, pl, z, pointer(buf), base, rounding)
+        end
+    elseif _nulterminated(source, len)
         GC.@preserve source begin
             return _finish_bigfloat(source, pos, code, pl, z, pointer(source, pos), base, rounding)
         end
     elseif _writable_contiguous(source)
-        nulpos = if _ends_on_delimiter(source, len, options)
+        nulpos = if _ends_on_delimiter(source, len, options, checkdelim)
             len
         else
-            _, _, strpl, _ = typeparser(DefaultConf{String}(), source, pos, len, b, code, poslen(pos, 0), options)
+            strpl = _bigfloat_poslen(source, pos, len, b, code, options, checkdelim)
             strpl.pos + strpl.len
         end
         if nulpos <= length(source)
@@ -126,24 +153,29 @@ function typeparser(::AbstractConf{BigFloat}, source, pos, len, b, code, pl, opt
                 @inbounds source[nulpos] = byte
             end
         else
-            _, _, strpl, _ = typeparser(DefaultConf{String}(), source, pos, len, b, code, poslen(pos, 0), options)
+            strpl = _bigfloat_poslen(source, pos, len, b, code, options, checkdelim)
             buf = _copy_nulterminated(source, strpl.pos, strpl.len)
             GC.@preserve buf begin
                 return _finish_bigfloat(source, pos, code, pl, z, pointer(buf), base, rounding)
             end
         end
     else
-        buf = _bigfloat_buffer(source, pos, len, b, code, options)
+        buf = _bigfloat_buffer(source, pos, len, b, code, options, checkdelim)
         GC.@preserve buf begin
             return _finish_bigfloat(source, pos, code, pl, z, pointer(buf), base, rounding)
         end
     end
 end
 
 @inline function typeparser(conf::AbstractConf{T}, source, pos, len, b, code, pl, options) where {T <: SupportedFloats}
+    return typeparser(conf, source, pos, len, b, code, pl, options, options.flags.checkdelim)
+end
+
+@inline function typeparser(conf::AbstractConf{T}, source, pos, len, b, code, pl, options, checkdelim::Bool) where {T <: SupportedFloats}
     # keep track of starting pos in case of invalid, we can rewind to start of parsing
     startpos = pos
     x = zero(T)
+    decimal = _effective_decimal(options, code, checkdelim)
     neg = b == UInt8('-')
     if neg || b == UInt8('+')
         pos += 1
@@ -155,7 +187,7 @@ end
         @goto done
     end
     b = peekbyte(source, pos)
-    if b != options.decimal && (b - UInt8('0')) > 0x09
+    if b != decimal && (b - UInt8('0')) > 0x09
         # character isn't a digit or decimal point, check for special values, otherwise INVALID
         if b == UInt8('n') || b == UInt8('N')
             pos += 1
@@ -267,7 +299,7 @@ end
     end
 
     # start parsing digits or decimal point; we start digits as UInt64(0) and can _widen type if needed
-    x, code, pos = parsedigits(conf, source, pos, len, b, code, options, UInt64(0), neg, startpos)
+    x, code, pos = parsedigits(conf, source, pos, len, b, code, options, decimal, UInt64(0), neg, startpos)
     if !isfinite(x)
         code |= SPECIAL_VALUE
     end
@@ -302,24 +334,24 @@ getx(x, f) = f === nothing ? x : nothing
 # statically compiled (`juliac --trim`) binary the widened instance doesn't exist, so
 # parsing a wide-digit float would fail at runtime. The `@noinline` wrappers keep each
 # ladder step compiling separately, which is what bounds base-case compilation.
-@noinline _parsedigits(conf::AbstractConf{T}, source, pos, len, b, code, options, digits::IntType, neg::Bool, startpos, overflow_invalid::Bool, ndigits::Int, f::F) where {T, IntType, F} =
-    parsedigits(conf, source, pos, len, b, code, options, digits, neg, startpos, overflow_invalid, ndigits, f)::Tuple{rettype(T), ReturnCode, Int}
+@noinline _parsedigits(conf::AbstractConf{T}, source, pos, len, b, code, options, decimal::UInt8, digits::IntType, neg::Bool, startpos, overflow_invalid::Bool, ndigits::Int, f::F) where {T, IntType, F} =
+    parsedigits(conf, source, pos, len, b, code, options, decimal, digits, neg, startpos, overflow_invalid, ndigits, f)::Tuple{rettype(T), ReturnCode, Int}
 
-@inline function parsedigits(conf::AbstractConf{T}, source, pos, len, b, code, options, digits::IntType, neg::Bool, startpos, overflow_invalid::Bool=false, ndigits::Int=0, f::F=nothing) where {T, IntType, F}
+@inline function parsedigits(conf::AbstractConf{T}, source, pos, len, b, code, options, decimal::UInt8, digits::IntType, neg::Bool, startpos, overflow_invalid::Bool=false, ndigits::Int=0, f::F=nothing) where {T, IntType, F}
     x = zero(T)
     anydigits = false
     has_groupmark = _has_groupmark(options, code)
     groupmark0 = something(options.groupmark, 0xff) - UInt8('0')
 
     # we already previously checked if `b` was decimal or a digit, so don't need to check explicitly again
-    if b != options.decimal
+    if b != decimal
         b -= UInt8('0')
         prev_b0 = b
         anydigits = b <= 0x09
         while true
             if b <= 0x09
                 if overflows(IntType) && digits > overflowval(IntType)
-                    return _parsedigits(conf, source, pos, len, b + UInt8('0'), code, options, _widen(digits), neg, startpos, overflow_invalid, ndigits, f)::Tuple{rettype(T), ReturnCode, Int}
+                    return _parsedigits(conf, source, pos, len, b + UInt8('0'), code, options, decimal, _widen(digits), neg, startpos, overflow_invalid, ndigits, f)::Tuple{rettype(T), ReturnCode, Int}
                 elseif ndigits > maxdigits(T)
                     # if input is way too big, just bail
                     fastseek!(source, startpos - 1)
@@ -361,7 +393,7 @@ getx(x, f) = f === nothing ? x : nothing
         # b wasn't a digit, so add back '0' to recover original Char value
         b += UInt8('0')
     end
-    if b == options.decimal
+    if b == decimal
         pos += 1
         incr!(source)
         if eof(source, pos, len)

src/ints.jl

@@ -98,15 +98,23 @@ overflowval(::Type{T}) where {T <: Integer} = div(typemax(T) - T(9), T(10))
     return pos, code, PosLen(pl.pos, pos - pl.pos), x
 end
 
-function typeparser(::AbstractConf{Number}, source, pos, len, b, code, pl, opts)
+function typeparser(conf::AbstractConf{Number}, source, pos, len, b, code, pl, opts)
+    return typeparser(conf, source, pos, len, b, code, pl, opts, opts.flags.checkdelim)
+end
+
+function typeparser(::AbstractConf{Number}, source, pos, len, b, code, pl, opts, checkdelim::Bool)
     x = Ref{Number}()
-    pos, code = parsenumber(source, pos, len, b, y -> (x[] = y), opts)
+    pos, code = parsenumber(source, pos, len, b, code, y -> (x[] = y), opts, checkdelim)
     return pos, code, PosLen(pl.pos, pos - pl.pos), isdefined(x, :x) ? x[] : (0::Number)
 end
 
 function parsenumber(source, pos, len, b, f::F, opts=OPTIONS) where {F}
+    return parsenumber(source, pos, len, b, SUCCESS, f, opts, opts.flags.checkdelim)
+end
+
+function parsenumber(source, pos, len, b, code, f::F, opts, checkdelim::Bool) where {F}
     startpos = pos
-    code = startcode = SUCCESS
+    startcode = code
     # begin parsing
     neg = b == UInt8('-')
     if neg || b == UInt8('+')
@@ -119,11 +127,12 @@ function parsenumber(source, pos, len, b, f::F, opts=OPTIONS) where {F}
     end
     b = peekbyte(source, pos)
     # parse rest of number
-    _, code, pos = parsedigits(DefaultConf{Number}(), source, pos, len, b, code, OPTIONS, Int64(0), neg, startpos, true, 0, f)
+    decimal = _effective_decimal(opts, code, checkdelim)
+    _, code, pos = parsedigits(DefaultConf{Number}(), source, pos, len, b, code, opts, decimal, Int64(0), neg, startpos, true, 0, f)
     if invalid(code)
         # by default, parsedigits only has up to Float64 precision; if we overflow
         # let's try BigFloat
-        pos, code, _, x = typeparser(DefaultConf{BigFloat}(), source, startpos, len, b, startcode, poslen(pos, 0), opts)
+        pos, code, _, x = typeparser(DefaultConf{BigFloat}(), source, startpos, len, b, startcode, poslen(pos, 0), opts, checkdelim)
         if ok(code)
             f(x)
         end

test/floats.jl

@@ -479,6 +479,57 @@ end
     end
 end
 
+@testset "decimal matches delimiter" begin
+    source_fns = (
+        identity,
+        s -> SubString("_" * s, 2),
+        s -> collect(codeunits(s)),
+        s -> view(collect(codeunits(s)), :),
+        IOBuffer,
+    )
+
+    for T in (Float16, Float32, Float64, BigFloat, Number)
+        expected_decimal = T === BigFloat ? BigFloat("1.2") : T === Number ? 1.2 : T(1.2)
+        for decimal in (',', '.')
+            unquoted = "1$(decimal)2$(decimal)3"
+            quoted = "\"1$(decimal)2\"$(decimal)3"
+            for make_source in source_fns
+                res = Parsers.xparse(T, make_source(unquoted); decimal=decimal, delim=decimal)
+                @test Parsers.ok(res.code)
+                @test Parsers.delimited(res.code)
+                @test !Parsers.quoted(res.code)
+                @test res.tlen == 2
+                @test res.val == one(T)
+
+                res = Parsers.xparse(T, make_source(quoted); decimal=decimal, delim=decimal)
+                @test Parsers.ok(res.code)
+                @test Parsers.delimited(res.code)
+                @test Parsers.quoted(res.code)
+                @test res.tlen == 6
+                @test res.val == expected_decimal
+            end
+        end
+
+        for make_source in source_fns
+            res = Parsers.xparse(T, make_source("1,2;3"); decimal=',', delim=';')
+            @test Parsers.ok(res.code)
+            @test Parsers.delimited(res.code)
+            @test res.tlen == 4
+            @test res.val == expected_decimal
+        end
+
+        options = Parsers.Options(decimal=',')
+        @test Parsers.parse(T, "1,2", options) == expected_decimal
+        @test Parsers.parse(T, collect(codeunits("1,2")), options) == expected_decimal
+        @test Parsers.parse(T, IOBuffer("1,2"), options) == expected_decimal
+    end
+
+    bytes = collect(codeunits("1,2;3"))
+    Parsers.xparse(BigFloat, bytes; decimal=',', delim=';')
+    @test bytes == codeunits("1,2;3")
+    @test Parsers.parse(BigFloat, collect(codeunits("1.")), Parsers.Options(delim='.')) == BigFloat(1)
+end
+
 @testset "BigFloats" begin
     res = Parsers.xparse(BigFloat, Vector(codeunits("1")), 1, 1)
     @test res.val == BigFloat(1)