diff --git a/src/connection.cpp b/src/connection.cpp index 388c28bb..c3e0f42c 100644 --- a/src/connection.cpp +++ b/src/connection.cpp @@ -306,6 +306,7 @@ PyObject* Connection_New(PyObject* pConnectString, bool fAutoCommit, long timeou cnxn->maxwrite = 0; cnxn->timeout = 0; cnxn->map_sqltype_to_converter = 0; + cnxn->readvar_initsize = 4096; cnxn->compat_diagrec_byte_length = false; cnxn->attrs_before = attrs_before_o.Detach(); @@ -1034,6 +1035,34 @@ static int Connection_setmaxwrite(PyObject* self, PyObject* value, void* closure return 0; } +static PyObject* Connection_getreadvarinitsize(PyObject* self, void* closure) +{ + UNUSED(closure); + + Connection* cnxn = Connection_Validate(self); + if (!cnxn) + return 0; + return PyLong_FromSsize_t(cnxn->readvar_initsize); +} + +static int Connection_setreadvarinitsize(PyObject* self, PyObject* value, void* closure) +{ + UNUSED(closure); + + Connection* cnxn = Connection_Validate(self); + if (!cnxn) + return -1; + Py_ssize_t v = PyLong_AsSsize_t(value); + if (v == -1 && PyErr_Occurred()) + return -1; + if (v < 0) + { + PyErr_SetString(PyExc_TypeError, "Cannot set readvar_initsize to a negative value."); + return -1; + } + cnxn->readvar_initsize = v; + return 0; +} static PyObject* Connection_gettimeout(PyObject* self, void* closure) { @@ -1534,6 +1563,8 @@ static PyGetSetDef Connection_getseters[] = { { "timeout", Connection_gettimeout, Connection_settimeout, "The timeout in seconds, zero means no timeout.", 0 }, { "maxwrite", Connection_getmaxwrite, Connection_setmaxwrite, "The maximum bytes to write before using SQLPutData.", 0 }, + { "readvar_initsize", Connection_getreadvarinitsize, Connection_setreadvarinitsize, + "The initial buffer size in bytes for reading values from variable-length columns.", 0 }, { "fetch_decimal_as_string", Connection_getfetchdecimalasstring, Connection_setfetchdecimalasstring, "If True, DECIMAL and NUMERIC values are fetched as strings using the legacy\n" "locale-aware path. If False (the default), values are fetched using a binary\n" diff --git a/src/connection.h b/src/connection.h index 3b4679fd..44a04f9a 100644 --- a/src/connection.h +++ b/src/connection.h @@ -45,6 +45,12 @@ struct Connection // The column size of datetime columns, obtained from SQLGetInfo(), used to determine the datetime precision. int datetime_precision; + // Initial buffer allocation size for ReadVarColumn + // Any integer greater than 0 overrides the default value of 4096 + // < 1 means use columnSize * cbElement + cbNullTeminator from the column descriptor, + // up to a ceiling of 32 MB. See https://github.com/mkleehammer/pyodbc/issues/1071. + Py_ssize_t readvar_initsize; + // The connection timeout in seconds. long timeout; diff --git a/src/getdata.cpp b/src/getdata.cpp index 734989ab..6b51c33b 100644 --- a/src/getdata.cpp +++ b/src/getdata.cpp @@ -93,8 +93,32 @@ static bool ReadVarColumn(Cursor* cur, Py_ssize_t iCol, SQLSMALLINT ctype, bool& const Py_ssize_t cbElement = (Py_ssize_t)(IsWideType(ctype) ? sizeof(uint16_t) : 1); const Py_ssize_t cbNullTerminator = IsBinaryType(ctype) ? 0 : cbElement; - // TODO: Make the initial allocation size configurable? - Py_ssize_t cbAllocated = 4096; + Py_ssize_t cbAllocated; + Py_ssize_t initsize = cur->cnxn->readvar_initsize; + if (initsize == 0) + { + // Use the column size from the descriptor so that a single SQLGetData + // call can return all data at once — required to work around Oracle BI + // ODBC drivers that loop forever when the buffer is smaller than the + // column's total data size. + Py_ssize_t columnSize = (Py_ssize_t)cur->colinfos[iCol].column_size; + + // columnSize is in characters for wide types, bytes for narrow. + // Multiply by cbElement to get bytes, add room for the null terminator. + cbAllocated = columnSize * cbElement + cbNullTerminator; + + // Clamp to something sane: if the driver reported 0 or an absurd value, + // fall back to a reasonable ceiling rather than allocating nothing or + // gigabytes blindly. + Py_ssize_t ceiling = 32 * 1024 * 1024; + if (cbAllocated <= 0 || cbAllocated > ceiling) + cbAllocated = ceiling; + } + else + { + cbAllocated = initsize; + } + Py_ssize_t cbUsed = 0; byte* pb = (byte*)PyMem_Malloc((size_t)cbAllocated); if (!pb) @@ -143,7 +167,7 @@ static bool ReadVarColumn(Cursor* cur, Py_ssize_t iCol, SQLSMALLINT ctype, bool& if (ret == SQL_SUCCESS_WITH_INFO) { // This means we read some data, but there is more. SQLGetData is very weird - it - // sets cbRead to the number of bytes we read *plus* the amount remaining. + // sets cbData to the number of bytes we read *plus* the amount remaining. Py_ssize_t cbRemaining = 0; // How many more bytes do we need to allocate, not including null? Py_ssize_t cbRead = 0; // How much did we just read, not including null? diff --git a/src/pyodbc.pyi b/src/pyodbc.pyi index da5e6938..c43c5f32 100644 --- a/src/pyodbc.pyi +++ b/src/pyodbc.pyi @@ -398,6 +398,15 @@ class Connection: def maxwrite(self, value: int) -> None: ... + @property + def readvar_initsize(self) -> int: + """The initial buffer size in bytes for reading values from variable-length columns.""" + ... + + @readvar_initsize.setter + def readvar_initsize(self, value: int) -> None: + ... + @property def searchescape(self) -> str: """The character for escaping search pattern characters like "%" and "_". diff --git a/tests/mysql_test.py b/tests/mysql_test.py index b8bfd0ba..732ae89c 100644 --- a/tests/mysql_test.py +++ b/tests/mysql_test.py @@ -68,17 +68,20 @@ def test_blob(cursor: pyodbc.Cursor): def _test_vartype(cursor, datatype): + assert cursor.connection.readvar_initsize == 4096 cursor.execute(f"create table t1(c1 {datatype}(4000))") + for initsize in [None, 1024 * 1024, 0]: + if initsize is not None: + cursor.connection.readvar_initsize = initsize + for length in [None, 0, 100, 1000, 4000]: + cursor.execute("delete from t1") - for length in [None, 0, 100, 1000, 4000]: - cursor.execute("delete from t1") - - encoding = (datatype in ('blob', 'varbinary')) and 'utf8' or None - value = _generate_str(length, encoding=encoding) + encoding = (datatype in ('blob', 'varbinary')) and 'utf8' or None + value = _generate_str(length, encoding=encoding) - cursor.execute("insert into t1 values(?)", value) - v = cursor.execute("select * from t1").fetchone()[0] - assert v == value + cursor.execute("insert into t1 values(?)", value) + v = cursor.execute("select * from t1").fetchone()[0] + assert v == value def test_char(cursor: pyodbc.Cursor): diff --git a/tests/postgresql_test.py b/tests/postgresql_test.py index 9da6e3ae..2b5e4bec 100644 --- a/tests/postgresql_test.py +++ b/tests/postgresql_test.py @@ -63,15 +63,19 @@ def _generate_str(length, encoding=None): def test_text(cursor: pyodbc.Cursor): cursor.execute("create table t1(col text)") + assert cursor.connection.readvar_initsize == 4096 # Two different read code paths exist based on the length. Using 100 and 4000 will ensure # both are tested. - for length in [None, 0, 100, 1000, 4000]: - cursor.execute("truncate table t1") - param = _generate_str(length) - cursor.execute("insert into t1 values (?)", param) - result = cursor.execute("select col from t1").fetchval() - assert result == param + for initsize in [None, 1024 * 1024, 0]: + if initsize is not None: + cursor.connection.readvar_initsize = initsize + for length in [None, 0, 100, 1000, 4000]: + cursor.execute("truncate table t1") + param = _generate_str(length) + cursor.execute("insert into t1 values (?)", param) + result = cursor.execute("select col from t1").fetchval() + assert result == param def test_text_many(cursor: pyodbc.Cursor): diff --git a/tests/sqlite_test.py b/tests/sqlite_test.py index 4e162312..65d43f3e 100644 --- a/tests/sqlite_test.py +++ b/tests/sqlite_test.py @@ -139,31 +139,36 @@ def _test_strtype(cursor: pyodbc.Cursor, sqltype, value, colsize=None): The implementation for string, Unicode, and binary tests. """ assert colsize is None or (value is None or colsize >= len(value)) + assert cursor.connection.readvar_initsize == 4096 - if colsize: - sql = "create table t1(s {}({}))".format(sqltype, colsize) - else: - sql = "create table t1(s {})".format(sqltype) + for initsize in [None, 1024 * 1024, 0]: + if initsize is not None: + cursor.connection.readvar_initsize = initsize + if colsize: + sql = "create table t1(s {}({}))".format(sqltype, colsize) + else: + sql = "create table t1(s {})".format(sqltype) - cursor.execute(sql) - cursor.execute("insert into t1 values(?)", value) - v = cursor.execute("select * from t1").fetchone()[0] - assert type(v) is type(value) + cursor.execute(sql) + cursor.execute("insert into t1 values(?)", value) + v = cursor.execute("select * from t1").fetchone()[0] + assert type(v) is type(value) - if value is not None: - assert len(v) == len(value) + if value is not None: + assert len(v) == len(value) - assert v == value + assert v == value - # Reported by Andy Hochhaus in the pyodbc group: In 2.1.7 and earlier, a hardcoded length of 255 was used to - # determine whether a parameter was bound as a SQL_VARCHAR or SQL_LONGVARCHAR. Apparently SQL Server chokes if - # we bind as a SQL_LONGVARCHAR and the target column size is 8000 or less, which is considers just SQL_VARCHAR. - # This means binding a 256 character value would cause problems if compared with a VARCHAR column under - # 8001. We now use SQLGetTypeInfo to determine the time to switch. - # - # [42000] [Microsoft][SQL Server Native Client 10.0][SQL Server]The data types varchar and text are incompatible in the equal to operator. + # Reported by Andy Hochhaus in the pyodbc group: In 2.1.7 and earlier, a hardcoded length of 255 was used to + # determine whether a parameter was bound as a SQL_VARCHAR or SQL_LONGVARCHAR. Apparently SQL Server chokes if + # we bind as a SQL_LONGVARCHAR and the target column size is 8000 or less, which is considers just SQL_VARCHAR. + # This means binding a 256 character value would cause problems if compared with a VARCHAR column under + # 8001. We now use SQLGetTypeInfo to determine the time to switch. + # + # [42000] [Microsoft][SQL Server Native Client 10.0][SQL Server]The data types varchar and text are incompatible in the equal to operator. - cursor.execute("select * from t1 where s=?", value) + cursor.execute("select * from t1 where s=?", value) + cursor.execute("drop table t1") def _test_strliketype(cursor: pyodbc.Cursor, sqltype, value, colsize=None): diff --git a/tests/sqlserver_test.py b/tests/sqlserver_test.py index 2b05f734..907015dc 100755 --- a/tests/sqlserver_test.py +++ b/tests/sqlserver_test.py @@ -241,37 +241,44 @@ def _test_vartype(cursor: pyodbc.Cursor, datatype): else: lengths = SMALL_FENCEPOST_SIZES - if datatype == 'text': - cursor.execute(f"create table t1(c1 {datatype})") - else: - maxlen = lengths[-1] - cursor.execute(f"create table t1(c1 {datatype}({maxlen}))") + assert cursor.connection.readvar_initsize == 4096 + for initsize in (None, 1024 * 1024, 0): + if initsize is not None: + cursor.connection.readvar_initsize = initsize - for length in lengths: + if datatype == 'text': + cursor.execute(f"create table t1(c1 {datatype})") + else: + maxlen = lengths[-1] + cursor.execute(f"create table t1(c1 {datatype}({maxlen}))") - # FreeTDS did not support SQLDescribeParam until version 1.5.16 (see ticket - # https://github.com/FreeTDS/freetds/issues/104), so pyodbc had to infer the - # SQL type from the Python value. None carries no type information, causing - # pyodbc to fall back to SQL_VARCHAR, which SQL Server rejects for binary - # columns. - if length is None and IS_FREETDS and is_binary and DRIVER_VERSION < (1, 5, 16): - continue + for length in lengths: - cursor.execute("delete from t1") + # FreeTDS did not support SQLDescribeParam until version 1.5.16 (see ticket + # https://github.com/FreeTDS/freetds/issues/104), so pyodbc had to infer the + # SQL type from the Python value. None carries no type information, causing + # pyodbc to fall back to SQL_VARCHAR, which SQL Server rejects for binary + # columns. + if length is None and IS_FREETDS and is_binary and DRIVER_VERSION < (1, 5, 16): + continue - value = _generate_str(length, encoding=encoding) + cursor.execute("delete from t1") - try: - cursor.execute("insert into t1 values(?)", value) - except pyodbc.Error as ex: - if value is None: - msg = f"{datatype} insert of NULL failed" - else: - msg = f'{datatype} insert failed: length={length} len={len(value)}' - raise Exception(msg) from ex - - v = cursor.execute("select * from t1").fetchone()[0] - assert v == value + value = _generate_str(length, encoding=encoding) + + try: + cursor.execute("insert into t1 values(?)", value) + except pyodbc.Error as ex: + if value is None: + msg = f"{datatype} insert of NULL failed" + else: + msg = f'{datatype} insert failed: length={length} len={len(value)}' + raise Exception(msg) from ex + + v = cursor.execute("select * from t1").fetchone()[0] + assert v == value + + cursor.execute("drop table t1") def _test_scalar(cursor: pyodbc.Cursor, datatype, values):