Make initial ReadVarColumn buffer size configurable

src/connection.cpp

@@ -306,6 +306,7 @@ PyObject* Connection_New(PyObject* pConnectString, bool fAutoCommit, long timeou
     cnxn->maxwrite     = 0;
     cnxn->timeout      = 0;
     cnxn->map_sqltype_to_converter = 0;
+    cnxn->readvar_initsize = 4096;
     cnxn->compat_diagrec_byte_length = false;
 
     cnxn->attrs_before = attrs_before_o.Detach();
@@ -1034,6 +1035,34 @@ static int Connection_setmaxwrite(PyObject* self, PyObject* value, void* closure
     return 0;
 }
 
+static PyObject* Connection_getreadvarinitsize(PyObject* self, void* closure)
+{
+    UNUSED(closure);
+
+    Connection* cnxn = Connection_Validate(self);
+    if (!cnxn)
+        return 0;
+    return PyLong_FromSsize_t(cnxn->readvar_initsize);
+}
+
+static int Connection_setreadvarinitsize(PyObject* self, PyObject* value, void* closure)
+{
+    UNUSED(closure);
+
+    Connection* cnxn = Connection_Validate(self);
+    if (!cnxn)
+        return -1;
+    Py_ssize_t v = PyLong_AsSsize_t(value);
+    if (v == -1 && PyErr_Occurred())
+        return -1;
+    if (v < 0)
+    {
+        PyErr_SetString(PyExc_TypeError, "Cannot set readvar_initsize to a negative value.");
+        return -1;
+    }
+    cnxn->readvar_initsize = v;
+    return 0;
+}
 
 static PyObject* Connection_gettimeout(PyObject* self, void* closure)
 {
@@ -1534,6 +1563,8 @@ static PyGetSetDef Connection_getseters[] = {
     { "timeout", Connection_gettimeout, Connection_settimeout,
       "The timeout in seconds, zero means no timeout.", 0 },
     { "maxwrite", Connection_getmaxwrite, Connection_setmaxwrite, "The maximum bytes to write before using SQLPutData.", 0 },
+    { "readvar_initsize", Connection_getreadvarinitsize, Connection_setreadvarinitsize,
+      "The initial buffer size in bytes for reading values from variable-length columns.", 0 },
     { "fetch_decimal_as_string", Connection_getfetchdecimalasstring, Connection_setfetchdecimalasstring,
       "If True, DECIMAL and NUMERIC values are fetched as strings using the legacy\n"
       "locale-aware path.  If False (the default), values are fetched using a binary\n"

src/connection.h

@@ -45,6 +45,12 @@ struct Connection
     // The column size of datetime columns, obtained from SQLGetInfo(), used to determine the datetime precision.
     int datetime_precision;
 
+    // Initial buffer allocation size for ReadVarColumn
+    // Any integer greater than 0 overrides the default value of 4096
+    // < 1 means use columnSize * cbElement + cbNullTeminator from the column descriptor,
+    // up to a ceiling of 32 MB. See https://github.com/mkleehammer/pyodbc/issues/1071.
+    Py_ssize_t readvar_initsize;
+
     // The connection timeout in seconds.
     long timeout;
 

src/getdata.cpp

@@ -93,8 +93,32 @@ static bool ReadVarColumn(Cursor* cur, Py_ssize_t iCol, SQLSMALLINT ctype, bool&
     const Py_ssize_t cbElement = (Py_ssize_t)(IsWideType(ctype) ? sizeof(uint16_t) : 1);
     const Py_ssize_t cbNullTerminator = IsBinaryType(ctype) ? 0 : cbElement;
 
-    // TODO: Make the initial allocation size configurable?
-    Py_ssize_t cbAllocated = 4096;
+    Py_ssize_t cbAllocated;
+    Py_ssize_t initsize = cur->cnxn->readvar_initsize;
+    if (initsize == 0)
+    {
+        // Use the column size from the descriptor so that a single SQLGetData
+        // call can return all data at once — required to work around Oracle BI
+        // ODBC drivers that loop forever when the buffer is smaller than the
+        // column's total data size.
+        Py_ssize_t columnSize = (Py_ssize_t)cur->colinfos[iCol].column_size;
+
+        // columnSize is in characters for wide types, bytes for narrow.
+        // Multiply by cbElement to get bytes, add room for the null terminator.
+        cbAllocated = columnSize * cbElement + cbNullTerminator;
+
+        // Clamp to something sane: if the driver reported 0 or an absurd value,
+        // fall back to a reasonable ceiling rather than allocating nothing or
+        // gigabytes blindly.
+        Py_ssize_t ceiling = 32 * 1024 * 1024;
+        if (cbAllocated <= 0 || cbAllocated > ceiling)
+            cbAllocated = ceiling;
+    }
+    else
+    {
+        cbAllocated = initsize;
+    }
+
     Py_ssize_t cbUsed = 0;
     byte* pb = (byte*)PyMem_Malloc((size_t)cbAllocated);
     if (!pb)
@@ -143,7 +167,7 @@ static bool ReadVarColumn(Cursor* cur, Py_ssize_t iCol, SQLSMALLINT ctype, bool&
         if (ret == SQL_SUCCESS_WITH_INFO)
         {
             // This means we read some data, but there is more.  SQLGetData is very weird - it
-            // sets cbRead to the number of bytes we read *plus* the amount remaining.
+            // sets cbData to the number of bytes we read *plus* the amount remaining.
 
             Py_ssize_t cbRemaining = 0; // How many more bytes do we need to allocate, not including null?
             Py_ssize_t cbRead = 0; // How much did we just read, not including null?

src/pyodbc.pyi

@@ -398,6 +398,15 @@ class Connection:
     def maxwrite(self, value: int) -> None:
         ...
 
+    @property
+    def readvar_initsize(self) -> int:
+        """The initial buffer size in bytes for reading values from variable-length columns."""
+        ...
+
+    @readvar_initsize.setter
+    def readvar_initsize(self, value: int) -> None:
+        ...
+
     @property
     def searchescape(self) -> str:
         """The character for escaping search pattern characters like "%" and "_".

tests/mysql_test.py

@@ -68,17 +68,20 @@ def test_blob(cursor: pyodbc.Cursor):
 
 
 def _test_vartype(cursor, datatype):
+    assert cursor.connection.readvar_initsize == 4096
     cursor.execute(f"create table t1(c1 {datatype}(4000))")
+    for initsize in [None, 1024 * 1024, 0]:
+        if initsize is not None:
+            cursor.connection.readvar_initsize = initsize
+        for length in [None, 0, 100, 1000, 4000]:
+            cursor.execute("delete from t1")
 
-    for length in [None, 0, 100, 1000, 4000]:
-        cursor.execute("delete from t1")
-
-        encoding = (datatype in ('blob', 'varbinary')) and 'utf8' or None
-        value = _generate_str(length, encoding=encoding)
+            encoding = (datatype in ('blob', 'varbinary')) and 'utf8' or None
+            value = _generate_str(length, encoding=encoding)
 
-        cursor.execute("insert into t1 values(?)", value)
-        v = cursor.execute("select * from t1").fetchone()[0]
-        assert v == value
+            cursor.execute("insert into t1 values(?)", value)
+            v = cursor.execute("select * from t1").fetchone()[0]
+            assert v == value
 
 
 def test_char(cursor: pyodbc.Cursor):

tests/postgresql_test.py

@@ -63,15 +63,19 @@ def _generate_str(length, encoding=None):
 
 def test_text(cursor: pyodbc.Cursor):
     cursor.execute("create table t1(col text)")
+    assert cursor.connection.readvar_initsize == 4096
 
     # Two different read code paths exist based on the length.  Using 100 and 4000 will ensure
     # both are tested.
-    for length in [None, 0, 100, 1000, 4000]:
-        cursor.execute("truncate table t1")
-        param = _generate_str(length)
-        cursor.execute("insert into t1 values (?)", param)
-        result = cursor.execute("select col from t1").fetchval()
-        assert result == param
+    for initsize in [None, 1024 * 1024, 0]:
+        if initsize is not None:
+            cursor.connection.readvar_initsize = initsize
+        for length in [None, 0, 100, 1000, 4000]:
+            cursor.execute("truncate table t1")
+            param = _generate_str(length)
+            cursor.execute("insert into t1 values (?)", param)
+            result = cursor.execute("select col from t1").fetchval()
+            assert result == param
 
 
 def test_text_many(cursor: pyodbc.Cursor):

tests/sqlite_test.py

@@ -139,31 +139,36 @@ def _test_strtype(cursor: pyodbc.Cursor, sqltype, value, colsize=None):
     The implementation for string, Unicode, and binary tests.
     """
     assert colsize is None or (value is None or colsize >= len(value))
+    assert cursor.connection.readvar_initsize == 4096
 
-    if colsize:
-        sql = "create table t1(s {}({}))".format(sqltype, colsize)
-    else:
-        sql = "create table t1(s {})".format(sqltype)
+    for initsize in [None, 1024 * 1024, 0]:
+        if initsize is not None:
+            cursor.connection.readvar_initsize = initsize
+        if colsize:
+            sql = "create table t1(s {}({}))".format(sqltype, colsize)
+        else:
+            sql = "create table t1(s {})".format(sqltype)
 
-    cursor.execute(sql)
-    cursor.execute("insert into t1 values(?)", value)
-    v = cursor.execute("select * from t1").fetchone()[0]
-    assert type(v) is type(value)
+        cursor.execute(sql)
+        cursor.execute("insert into t1 values(?)", value)
+        v = cursor.execute("select * from t1").fetchone()[0]
+        assert type(v) is type(value)
 
-    if value is not None:
-        assert len(v) == len(value)
+        if value is not None:
+            assert len(v) == len(value)
 
-    assert v == value
+        assert v == value
 
-    # Reported by Andy Hochhaus in the pyodbc group: In 2.1.7 and earlier, a hardcoded length of 255 was used to
-    # determine whether a parameter was bound as a SQL_VARCHAR or SQL_LONGVARCHAR.  Apparently SQL Server chokes if
-    # we bind as a SQL_LONGVARCHAR and the target column size is 8000 or less, which is considers just SQL_VARCHAR.
-    # This means binding a 256 character value would cause problems if compared with a VARCHAR column under
-    # 8001. We now use SQLGetTypeInfo to determine the time to switch.
-    #
-    # [42000] [Microsoft][SQL Server Native Client 10.0][SQL Server]The data types varchar and text are incompatible in the equal to operator.
+        # Reported by Andy Hochhaus in the pyodbc group: In 2.1.7 and earlier, a hardcoded length of 255 was used to
+        # determine whether a parameter was bound as a SQL_VARCHAR or SQL_LONGVARCHAR.  Apparently SQL Server chokes if
+        # we bind as a SQL_LONGVARCHAR and the target column size is 8000 or less, which is considers just SQL_VARCHAR.
+        # This means binding a 256 character value would cause problems if compared with a VARCHAR column under
+        # 8001. We now use SQLGetTypeInfo to determine the time to switch.
+        #
+        # [42000] [Microsoft][SQL Server Native Client 10.0][SQL Server]The data types varchar and text are incompatible in the equal to operator.
 
-    cursor.execute("select * from t1 where s=?", value)
+        cursor.execute("select * from t1 where s=?", value)
+        cursor.execute("drop table t1")
 
 
 def _test_strliketype(cursor: pyodbc.Cursor, sqltype, value, colsize=None):

tests/sqlserver_test.py

@@ -241,37 +241,44 @@ def _test_vartype(cursor: pyodbc.Cursor, datatype):
     else:
         lengths = SMALL_FENCEPOST_SIZES
 
-    if datatype == 'text':
-        cursor.execute(f"create table t1(c1 {datatype})")
-    else:
-        maxlen = lengths[-1]
-        cursor.execute(f"create table t1(c1 {datatype}({maxlen}))")
+    assert cursor.connection.readvar_initsize == 4096
+    for initsize in (None, 1024 * 1024, 0):
+        if initsize is not None:
+            cursor.connection.readvar_initsize = initsize
 
-    for length in lengths:
+        if datatype == 'text':
+            cursor.execute(f"create table t1(c1 {datatype})")
+        else:
+            maxlen = lengths[-1]
+            cursor.execute(f"create table t1(c1 {datatype}({maxlen}))")
 
-        # FreeTDS did not support SQLDescribeParam until version 1.5.16 (see ticket
-        # https://github.com/FreeTDS/freetds/issues/104), so pyodbc had to infer the
-        # SQL type from the Python value. None carries no type information, causing
-        # pyodbc to fall back to SQL_VARCHAR, which SQL Server rejects for binary
-        # columns.
-        if length is None and IS_FREETDS and is_binary and DRIVER_VERSION < (1, 5, 16):
-            continue
+        for length in lengths:
 
-        cursor.execute("delete from t1")
+            # FreeTDS did not support SQLDescribeParam until version 1.5.16 (see ticket
+            # https://github.com/FreeTDS/freetds/issues/104), so pyodbc had to infer the
+            # SQL type from the Python value. None carries no type information, causing
+            # pyodbc to fall back to SQL_VARCHAR, which SQL Server rejects for binary
+            # columns.
+            if length is None and IS_FREETDS and is_binary and DRIVER_VERSION < (1, 5, 16):
+                continue
 
-        value = _generate_str(length, encoding=encoding)
+            cursor.execute("delete from t1")
 
-        try:
-            cursor.execute("insert into t1 values(?)", value)
-        except pyodbc.Error as ex:
-            if value is None:
-                msg = f"{datatype} insert of NULL failed"
-            else:
-                msg = f'{datatype} insert failed: length={length} len={len(value)}'
-            raise Exception(msg) from ex
-
-        v = cursor.execute("select * from t1").fetchone()[0]
-        assert v == value
+            value = _generate_str(length, encoding=encoding)
+
+            try:
+                cursor.execute("insert into t1 values(?)", value)
+            except pyodbc.Error as ex:
+                if value is None:
+                    msg = f"{datatype} insert of NULL failed"
+                else:
+                    msg = f'{datatype} insert failed: length={length} len={len(value)}'
+                raise Exception(msg) from ex
+
+            v = cursor.execute("select * from t1").fetchone()[0]
+            assert v == value
+
+        cursor.execute("drop table t1")
 
 
 def _test_scalar(cursor: pyodbc.Cursor, datatype, values):