Use SQL_NUMERIC_STRUCT by default for DECIMAL/NUMERIC values (#1470)

POTENTIALLY BREAKING: this changes the default behavior for fetching DECIMAL and
NUMERIC values for every connection on upgrade.

DECIMAL/NUMERIC values are now fetched via the binary SQL_NUMERIC_STRUCT
(SQL_C_NUMERIC) path by default, instead of being read as text and parsed -- the
old path guessed the locale decimal separator and produced wrong values in some
locales (#753). The binary path is locale-independent and also helps TVP numerics
(#996).

Opt out / restore the legacy string-based path per connection with:

    connection.fetch_decimal_as_string = True

The string path is also used automatically as a fallback when the column scale is
outside [0, 127] (e.g. PostgreSQL's negative-scale reporting) or when descriptor
setup fails. Also fixes a missing-argument bug in the DecimalFromText fallback.

Fixes #753. Helps #996.

Note for release notes: call this out as potentially breaking, document the
fetch_decimal_as_string bypass, and note the historical driver-compatibility
caution around SQL_NUMERIC_STRUCT.

Author: bkline

src/connection.cpp

@@ -309,6 +309,7 @@ PyObject* Connection_New(PyObject* pConnectString, bool fAutoCommit, long timeou
     cnxn->compat_diagrec_byte_length = false;
 
     cnxn->attrs_before = attrs_before_o.Detach();
+    cnxn->fetch_decimal_as_string = false;
 
     // This is an inefficient default, but should work all the time.  When we are offered
     // single-byte text we don't actually know what the encoding is.  For example, with SQL
@@ -1082,6 +1083,37 @@ static int Connection_settimeout(PyObject* self, PyObject* value, void* closure)
     return 0;
 }
 
+static PyObject* Connection_getfetchdecimalasstring(PyObject* self, void* closure)
+{
+    UNUSED(closure);
+
+    Connection* cnxn = Connection_Validate(self);
+    if (!cnxn)
+        return 0;
+
+    PyObject* result = cnxn->fetch_decimal_as_string ? Py_True : Py_False;
+    Py_INCREF(result);
+    return result;
+}
+
+static int Connection_setfetchdecimalasstring(PyObject* self, PyObject* value, void* closure)
+{
+    UNUSED(closure);
+
+    Connection* cnxn = Connection_Validate(self);
+    if (!cnxn)
+        return -1;
+
+    if (value == 0)
+    {
+        PyErr_SetString(PyExc_TypeError, "Cannot delete the fetch_decimal_as_string attribute.");
+        return -1;
+    }
+
+    cnxn->fetch_decimal_as_string = PyObject_IsTrue(value);
+    return 0;
+}
+
 static PyObject* Connection_getcompat_diagrec_byte_length(PyObject* self, void* closure)
 {
     return PyBool_FromLong(((Connection*)self)->compat_diagrec_byte_length);
@@ -1502,6 +1534,10 @@ static PyGetSetDef Connection_getseters[] = {
     { "timeout", Connection_gettimeout, Connection_settimeout,
       "The timeout in seconds, zero means no timeout.", 0 },
     { "maxwrite", Connection_getmaxwrite, Connection_setmaxwrite, "The maximum bytes to write before using SQLPutData.", 0 },
+    { "fetch_decimal_as_string", Connection_getfetchdecimalasstring, Connection_setfetchdecimalasstring,
+      "If True, DECIMAL and NUMERIC values are fetched as strings using the legacy\n"
+      "locale-aware path.  If False (the default), values are fetched using a binary\n"
+      "representation that is not affected by the locale.", 0 },
     { "compat_diagrec_byte_length", Connection_getcompat_diagrec_byte_length, Connection_setcompat_diagrec_byte_length,
       "If True, the driver reports byte length instead of character length in SQLGetDiagRecW().", 0 },
     { 0 }

src/connection.h

@@ -39,6 +39,9 @@ struct Connection
     // to insert NULLs into binary columns.
     bool supports_describeparam;
 
+    // Set to true if the driver doesn't handle SQL_NUMERIC_STRUCT properly.
+    bool fetch_decimal_as_string;
+
     // The column size of datetime columns, obtained from SQLGetInfo(), used to determine the datetime precision.
     int datetime_precision;
 

src/cursor.cpp

@@ -543,6 +543,53 @@ bool InitColumnInfo(Cursor* cursor, SQLUSMALLINT iCol, ColumnInfo* pinfo)
         pinfo->is_unsigned = false;
     }
 
+    // For a NUMERIC column, determine how we will fetch the values.
+    pinfo->scale = DecimalDigits;
+    pinfo->use_decimal_binary = false;
+    switch (pinfo->sql_type)
+    {
+    case SQL_DECIMAL:
+    case SQL_NUMERIC:
+    case SQL_DB2_DECFLOAT:
+
+        // It is puzzling that ODBC abandons support for scale values between 128 and 255
+        // by using a signed byte so that it can support negative scale values, even though
+        // the SQL standard only allows values between zero and precision. Not many databases
+        // support negative scale. PostgreSQL is one, and its ODBC driver currently (version
+        // 17.00.0007) reports that scale is 2046 for SELECT CAST('1234500' AS NUMERIC(10,-2))
+        // which doesn't make much sense (to me, anyway). So I'm going to follow the lead of
+        // the SQL standard and fall back on fetching the value from the driver as a string
+        // when scale does not fall in the range 0..127.
+        if (!cursor->cnxn->fetch_decimal_as_string && pinfo->scale >= 0 && pinfo->scale <= 127)
+        {
+            // Set up the ARD once for this column so SQLGetData fills a SQL_NUMERIC_STRUCT.
+            // These settings persist for all rows on this statement handle.
+            SQLHDESC hDesc = NULL;
+            SQLRETURN descRet;
+
+            Py_BEGIN_ALLOW_THREADS
+            descRet = SQLGetStmtAttr(cursor->hstmt, SQL_ATTR_APP_ROW_DESC, &hDesc, 0, NULL);
+            Py_END_ALLOW_THREADS
+
+            if (SQL_SUCCEEDED(descRet))
+            {
+                SQLULEN precision = pinfo->column_size;
+                SQLSMALLINT scale = pinfo->scale;
+                SQLSMALLINT i = (SQLSMALLINT)iCol;
+
+                Py_BEGIN_ALLOW_THREADS
+                // SQL_DESC_TYPE must be set first.
+                descRet =
+                    SQLSetDescField(hDesc, i, SQL_DESC_TYPE, (void*)SQL_C_NUMERIC, 0) == SQL_SUCCESS &&
+                    SQLSetDescField(hDesc, i, SQL_DESC_PRECISION, (void*)precision, 0) == SQL_SUCCESS &&
+                    SQLSetDescField(hDesc, i, SQL_DESC_SCALE, (void*)scale, 0) == SQL_SUCCESS
+                    ? SQL_SUCCESS : SQL_ERROR;
+                Py_END_ALLOW_THREADS
+
+                pinfo->use_decimal_binary = SQL_SUCCEEDED(descRet);
+            }
+        }
+    }
     return true;
 }
 

src/cursor.h

@@ -31,6 +31,13 @@ struct ColumnInfo
     // of the integer types are the same size whether signed and unsigned, so we can allocate memory ahead of time
     // without knowing this.  We use this during the fetch when converting to a Python integer or long.
     bool is_unsigned;
+
+    // For SQL_DECIMAL/SQL_NUMERIC columns: scale from SQLDescribeCol, and whether
+    // the binary (SQL_C_NUMERIC) fetch path can be used for this column.
+    // use_decimal_binary is false if scale is outside [0, 127] (doesn't fit in
+    // SQL_NUMERIC_STRUCT.scale or is negative) or if the driver failed ARD setup.
+    SQLSMALLINT scale;
+    bool use_decimal_binary;
 };
 
 struct ParamInfo
@@ -117,10 +124,10 @@ struct Cursor
 
     // Parameter set array (used with executemany)
     unsigned char *paramArray;
-    
+
     // Whether to use fast executemany with parameter arrays and other optimisations
     char fastexecmany;
-    
+
     // The list of information for setinputsizes().
     PyObject *inputsizes;
 

src/decimal.cpp

@@ -109,7 +109,6 @@ bool SetDecimalPoint(PyObject* pNew)
     return true;
 }
 
-
 PyObject* DecimalFromText(const TextEnc& enc, const byte* pb, Py_ssize_t cb)
 {
     // Creates a Decimal object from a text buffer.
@@ -131,11 +130,53 @@ PyObject* DecimalFromText(const TextEnc& enc, const byte* pb, Py_ssize_t cb)
 
     if (pLocaleDecimalEscaped)
     {
-        Object c2(PyObject_CallFunctionObjArgs(re_sub, pLocaleDecimalEscaped, pDecimalPoint, 0));
+        Object c2(PyObject_CallFunctionObjArgs(re_sub, pLocaleDecimalEscaped, pDecimalPoint, cleaned.Get(), 0));
         if (!c2)
             return 0;
         cleaned.Attach(c2.Detach());
     }
 
     return PyObject_CallFunctionObjArgs(decimal, cleaned.Get(), 0);
 }
+
+PyObject* DecimalFromNumericStruct(const SQL_NUMERIC_STRUCT& num)
+{
+    // Avoid the problems introduced by locale-specific delimiters in numeric strings
+    // by having the driver provide the value using a well-defined binary structure.
+
+    // Convert the little-endian (1) unsigned (0) magnitude to a Python int.
+    PyObject* magnitude = _PyLong_FromByteArray(num.val, SQL_MAX_NUMERIC_LEN, 1, 0);
+    if (!magnitude)
+        return NULL;
+
+    // Convert magnitude to a tuple of decimal digits via str().
+    Object magStr(PyObject_Str(magnitude));
+    Py_DECREF(magnitude);
+    if (!magStr)
+        return NULL;
+
+    Py_ssize_t nDigits = PyUnicode_GET_LENGTH(magStr.Get());
+    Object digitTuple(PyTuple_New(nDigits));
+    if (!digitTuple)
+        return NULL;
+
+    for (Py_ssize_t i = 0; i < nDigits; i++)
+    {
+        Py_UCS4 ch = PyUnicode_READ_CHAR(magStr.Get(), i);
+        PyObject* digit = PyLong_FromLong((long)(ch - '0'));
+        if (!digit)
+            return NULL;
+        PyTuple_SET_ITEM(digitTuple.Get(), i, digit);
+    }
+
+    // Decimal sign: 0 = positive, 1 = negative (opposite of ODBC convention).
+    int decimalSign = (num.sign == 1) ? 0 : 1;
+    long exponent = -(long)num.scale;
+
+    // Decimal((sign, (d, d, ...), exponent))
+    Object args(Py_BuildValue("((iOl))", decimalSign, digitTuple.Get(), exponent));
+    if (!args)
+        return NULL;
+
+    return PyObject_CallObject(decimal, args.Get());
+}

src/decimal.h

@@ -5,3 +5,4 @@ PyObject* GetDecimalPoint();
 bool SetDecimalPoint(PyObject* pNew);
 
 PyObject* DecimalFromText(const TextEnc& enc, const byte* pb, Py_ssize_t cb);
+PyObject* DecimalFromNumericStruct(const SQL_NUMERIC_STRUCT& num);

src/getdata.cpp

@@ -326,8 +326,27 @@ static PyObject* GetDataUser(Cursor* cur, Py_ssize_t iCol, PyObject* func)
     return result;
 }
 
+static PyObject* GetDataDecimalBinary(Cursor* cur, Py_ssize_t iCol)
+{
+    SQL_NUMERIC_STRUCT numStruct;
+    SQLLEN cbFetched = 0;
+    SQLRETURN ret;
+    SQLUSMALLINT i = (SQLUSMALLINT)(iCol + 1);
+
+    Py_BEGIN_ALLOW_THREADS
+    ret = SQLGetData(cur->hstmt, i, SQL_ARD_TYPE, &numStruct, sizeof(numStruct), &cbFetched);
+    Py_END_ALLOW_THREADS
+
+    if (!SQL_SUCCEEDED(ret))
+        return NULL;  // no Python exception — caller falls back to string path
 
-static PyObject* GetDataDecimal(Cursor* cur, Py_ssize_t iCol)
+    if (cbFetched == SQL_NULL_DATA)
+        Py_RETURN_NONE;
+
+    return DecimalFromNumericStruct(numStruct);
+}
+
+static PyObject* GetDataDecimalString(Cursor* cur, Py_ssize_t iCol)
 {
     // The SQL_NUMERIC_STRUCT support is hopeless (SQL Server ignores scale on input parameters
     // and output columns, Oracle does something else weird, and many drivers don't support it
@@ -731,7 +750,16 @@ PyObject* GetData(Cursor* cur, Py_ssize_t iCol)
     case SQL_DECIMAL:
     case SQL_NUMERIC:
     case SQL_DB2_DECFLOAT:
-        return GetDataDecimal(cur, iCol);
+        if (cur->colinfos[iCol].use_decimal_binary)
+        {
+            PyObject* obj = GetDataDecimalBinary(cur, iCol);
+            if (obj != NULL)
+                return obj;
+            if (PyErr_Occurred())
+                return NULL;
+            // SQLGetData failed without a Python exception — fall through.
+        }
+        return GetDataDecimalString(cur, iCol);
 
     case SQL_BIT:
         return GetDataBit(cur, iCol);

src/pyodbc.pyi

@@ -361,6 +361,17 @@ class Connection:
         """Returns True if the connection is closed, False otherwise."""
         ...
 
+    @property
+    def fetch_decimal_as_string(self) -> bool:
+        """If True, DECIMAL and NUMERIC values will be fetched as strings using the
+        legacy local-aware path.  If False (the default), values are fetched using a
+        binary representation that is not affected by the locale."""
+        ...
+
+    @fetch_decimal_as_string.setter
+    def fetch_decimal_as_string(self, value: bool) -> None:
+        ...
+
     @property
     def hdbc(self) -> ctypes.c_void_p | None:
         """ODBC handle for the connection."""

tests/mysql_test.py

@@ -118,13 +118,15 @@ def test_decimal(cursor: pyodbc.Cursor):
         ('-10.0010', '19,4')
     ]
 
-    for value, prec in tests:
-        value = Decimal(value)
-        cursor.execute("drop table if exists t1")
-        cursor.execute(f"create table t1(c1 numeric({prec}))")
-        cursor.execute("insert into t1 values (?)", value)
-        v = cursor.execute("select c1 from t1").fetchone()[0]
-        assert v == value
+    for mode in (True, False):
+        cursor.connection.fetch_decimal_as_string = mode
+        for value, prec in tests:
+            value = Decimal(value)
+            cursor.execute("drop table if exists t1")
+            cursor.execute(f"create table t1(c1 numeric({prec}))")
+            cursor.execute("insert into t1 values (?)", value)
+            v = cursor.execute("select c1 from t1").fetchone()[0]
+            assert v == value
 
 
 def test_multiple_bindings(cursor: pyodbc.Cursor):

tests/postgresql_test.py

@@ -168,11 +168,13 @@ def test_decimal(cursor: pyodbc.Cursor):
     params = [Decimal(n) for n in "-1000.10 -1234.56 -1 0 1 1000.10 1234.56 100010 123456789.21".split()]
     params.append(None)
 
-    for param in params:
-        cursor.execute("truncate table t1")
-        cursor.execute("insert into t1 values (?)", param)
-        result = cursor.execute("select col from t1").fetchval()
-        assert result == param
+    for mode in (True, False):
+        cursor.connection.fetch_decimal_as_string = mode
+        for param in params:
+            cursor.execute("truncate table t1")
+            cursor.execute("insert into t1 values (?)", param)
+            result = cursor.execute("select col from t1").fetchval()
+            assert result == param
 
 
 def test_numeric(cursor: pyodbc.Cursor):
@@ -182,11 +184,13 @@ def test_numeric(cursor: pyodbc.Cursor):
     params = [Decimal(n) for n in "-1234.56  -1  0  1  1234.56  123456789.21".split()]
     params.append(None)
 
-    for param in params:
-        cursor.execute("truncate table t1")
-        cursor.execute("insert into t1 values (?)", param)
-        result = cursor.execute("select col from t1").fetchval()
-        assert result == param
+    for mode in (True, False):
+        cursor.connection.fetch_decimal_as_string = mode
+        for param in params:
+            cursor.execute("truncate table t1")
+            cursor.execute("insert into t1 values (?)", param)
+            result = cursor.execute("select col from t1").fetchval()
+            assert result == param
 
 
 def test_maxwrite(cursor: pyodbc.Cursor):