diff --git a/src/connection.cpp b/src/connection.cpp index c3e0f42c..d923d708 100644 --- a/src/connection.cpp +++ b/src/connection.cpp @@ -306,6 +306,7 @@ PyObject* Connection_New(PyObject* pConnectString, bool fAutoCommit, long timeou cnxn->maxwrite = 0; cnxn->timeout = 0; cnxn->map_sqltype_to_converter = 0; + cnxn->preserve_tzoffsets = false; cnxn->readvar_initsize = 4096; cnxn->compat_diagrec_byte_length = false; @@ -931,6 +932,37 @@ static int Connection_setautocommit(PyObject* self, PyObject* value, void* closu return 0; } +PyObject* Connection_getpreservetzoffsets(PyObject* self, void* closure) +{ + UNUSED(closure); + + Connection* cnxn = Connection_Validate(self); + if (!cnxn) + return 0; + + PyObject* result = cnxn->preserve_tzoffsets ? Py_True : Py_False; + Py_INCREF(result); + return result; +} + +static int Connection_setpreservetzoffsets(PyObject* self, PyObject* value, void* closure) +{ + UNUSED(closure); + + Connection* cnxn = Connection_Validate(self); + if (!cnxn) + return -1; + + if (value == 0) + { + PyErr_SetString(PyExc_TypeError, "Cannot delete the preserve_tzoffsets attribute."); + return -1; + } + + cnxn->preserve_tzoffsets = PyObject_IsTrue(value) ? true : false; + + return 0; +} static PyObject* Connection_getclosed(PyObject* self, void* closure) { @@ -1560,6 +1592,8 @@ static PyGetSetDef Connection_getseters[] = { "SQLGetInfo(SQL_SEARCH_PATTERN_ESCAPE). These are driver specific.", 0 }, { "autocommit", Connection_getautocommit, Connection_setautocommit, "Returns True if the connection is in autocommit mode; False otherwise.", 0 }, + { "preserve_tzoffsets", Connection_getpreservetzoffsets, Connection_setpreservetzoffsets, + "Set to True to opt in for preserving time zone offset in parameters.", 0 }, { "timeout", Connection_gettimeout, Connection_settimeout, "The timeout in seconds, zero means no timeout.", 0 }, { "maxwrite", Connection_getmaxwrite, Connection_setmaxwrite, "The maximum bytes to write before using SQLPutData.", 0 }, diff --git a/src/connection.h b/src/connection.h index 44a04f9a..3543d7a5 100644 --- a/src/connection.h +++ b/src/connection.h @@ -45,6 +45,9 @@ struct Connection // The column size of datetime columns, obtained from SQLGetInfo(), used to determine the datetime precision. int datetime_precision; + // Opt-in flag for preserving time zone offset with non-naïve datetime objects. + bool preserve_tzoffsets; + // Initial buffer allocation size for ReadVarColumn // Any integer greater than 0 overrides the default value of 4096 // < 1 means use columnSize * cbElement + cbNullTeminator from the column descriptor, diff --git a/src/dbspecific.h b/src/dbspecific.h index 707247f5..43cf24b4 100644 --- a/src/dbspecific.h +++ b/src/dbspecific.h @@ -16,6 +16,7 @@ #define SQL_DB2_DECFLOAT -360 // IBM DB/2 DECFLOAT type #define SQL_DB2_XML -370 // IBM DB/2 XML type #define SQL_SS_TIME2 -154 // SQL Server 2008 time type +#define SQL_SS_TIMESTAMPOFFSET -155 // a misnomer; it's not an offset, it's a TIMESTAMP *with* an offset #define SQL_DB2_BLOB -98 // IBM DB/2 BLOB type struct SQL_SS_TIME2_STRUCT diff --git a/src/params.cpp b/src/params.cpp index 5355fcaa..1878a947 100644 --- a/src/params.cpp +++ b/src/params.cpp @@ -25,6 +25,7 @@ #include "row.h" #include +#define TZAWARE_STRING_LENGTH 32 // "YYYY-MM-DD HH:MM:SS.999999+00:00" inline Connection* GetConnection(Cursor* cursor) { @@ -37,8 +38,105 @@ struct DAEParam SQLLEN maxlen; }; +// Does this datetime object have time zone offset information? +static bool _datetime_is_tzaware(PyObject* dt) +{ +#if PY_VERSION_HEX >= 0x030A0000 + // Fast path: direct struct access, available from Python 3.10. + PyObject* tzinfo = PyDateTime_DATE_GET_TZINFO(dt); // borrowed ref + if (tzinfo == Py_None) + return false; + PyObject* offset = PyObject_CallMethod(tzinfo, "utcoffset", "O", dt); +#else + // Pre-3.10: fetch tzinfo via attribute lookup. + PyObject* tzinfo = PyObject_GetAttrString(dt, "tzinfo"); // new ref + if (tzinfo == nullptr || tzinfo == Py_None) + { + Py_XDECREF(tzinfo); + return false; + } + PyObject* offset = PyObject_CallMethod(tzinfo, "utcoffset", "O", dt); + Py_DECREF(tzinfo); +#endif + if (offset == nullptr) + { + PyErr_Clear(); + return false; + } + bool aware = (offset != Py_None); + Py_DECREF(offset); + return aware; +} -static int DetectCType(PyObject *cell, ParamInfo *pi) +// Custom string (not all back ends accept the output of dt.isoformat()). +static char* _make_tzaware_string(PyObject* dt) +{ + int year = PyDateTime_GET_YEAR(dt); + int month = PyDateTime_GET_MONTH(dt); + int day = PyDateTime_GET_DAY(dt); + int hour = PyDateTime_DATE_GET_HOUR(dt); + int minute = PyDateTime_DATE_GET_MINUTE(dt); + int second = PyDateTime_DATE_GET_SECOND(dt); + int us = PyDateTime_DATE_GET_MICROSECOND(dt); + int tz_hours = 0; + int tz_minutes = 0; + char sign = '+'; + + PyObject* tzinfo = PyObject_GetAttrString(dt, "tzinfo"); + if (tzinfo == nullptr) + return nullptr; + + if (tzinfo != Py_None) + { + PyObject* offset = PyObject_CallMethod(tzinfo, "utcoffset", "O", dt); + Py_DECREF(tzinfo); + if (offset == nullptr) + return nullptr; + + if (offset != Py_None) + { + PyObject* total_seconds_obj = PyObject_CallMethod(offset, "total_seconds", nullptr); + Py_DECREF(offset); + if (total_seconds_obj == nullptr) + return nullptr; + + double total_seconds = PyFloat_AsDouble(total_seconds_obj); + Py_DECREF(total_seconds_obj); + if (total_seconds == -1.0 && PyErr_Occurred()) + return nullptr; + + sign = (total_seconds < 0) ? '-' : '+'; + int abs_secs = (int)fabs(total_seconds); + tz_hours = abs_secs / 3600; + tz_minutes = (abs_secs % 3600) / 60; + + if (tz_hours > 23 || tz_minutes > 59) { + PyErr_SetString(PyExc_ValueError, + "datetime tzinfo.utcoffset() returned an invalid value"); + return nullptr; + } + } + else + Py_DECREF(offset); + } + else + Py_DECREF(tzinfo); + + char buf[TZAWARE_STRING_LENGTH + 1]; + PyOS_snprintf(buf, sizeof buf, "%04d-%02d-%02d %02d:%02d:%02d.%06d%c%02d:%02d", + year, month, day, + hour, minute, second, us, + sign, tz_hours, tz_minutes); + char* p = (char*)PyMem_Malloc(TZAWARE_STRING_LENGTH); + if (p == nullptr) { + PyErr_NoMemory(); + return nullptr; + } + memcpy(p, buf, TZAWARE_STRING_LENGTH); + return p; +} + +static int DetectCType(PyObject *cell, ParamInfo *pi, bool preserve_tzoffsets) { // Detects and sets the appropriate C type to use for binding the specified Python object. // Also sets the buffer length to use. Returns false if unsuccessful. @@ -96,6 +194,15 @@ static int DetectCType(PyObject *cell, ParamInfo *pi) Type_DateTime: pi->ValueType = SQL_C_TYPE_TIMESTAMP; pi->BufferLength = sizeof(SQL_TIMESTAMP_STRUCT); + if (preserve_tzoffsets) + { + if ((cell != Py_None && _datetime_is_tzaware(cell)) || pi->ParameterType == SQL_SS_TIMESTAMPOFFSET) + { + pi->ParameterType = SQL_VARCHAR; + pi->ValueType = SQL_C_CHAR; + pi->BufferLength = TZAWARE_STRING_LENGTH; + } + } } else if (PyDate_Check(cell)) { @@ -163,6 +270,7 @@ static int DetectCType(PyObject *cell, ParamInfo *pi) case SQL_TYPE_TIME: goto Type_Time; case SQL_TYPE_TIMESTAMP: + case SQL_SS_TIMESTAMPOFFSET: goto Type_DateTime; case SQL_GUID: goto Type_UUID; @@ -336,24 +444,37 @@ static int PyToCType(Cursor *cur, unsigned char **outbuf, PyObject *cell, ParamI } else if (PyDateTime_Check(cell)) { - if (pi->ValueType != SQL_C_TYPE_TIMESTAMP) + if (pi->ValueType == SQL_C_TYPE_TIMESTAMP) + { + SQL_TIMESTAMP_STRUCT *pts = (SQL_TIMESTAMP_STRUCT*)*outbuf; + pts->year = PyDateTime_GET_YEAR(cell); + pts->month = PyDateTime_GET_MONTH(cell); + pts->day = PyDateTime_GET_DAY(cell); + pts->hour = PyDateTime_DATE_GET_HOUR(cell); + pts->minute = PyDateTime_DATE_GET_MINUTE(cell); + pts->second = PyDateTime_DATE_GET_SECOND(cell); + + // Truncate the fraction according to precision + size_t digits = min(9, pi->DecimalDigits); + long fast_pow10[] = {1,10,100,1000,10000,100000,1000000,10000000,100000000,1000000000}; + SQLUINTEGER milliseconds = PyDateTime_DATE_GET_MICROSECOND(cell) * 1000; + pts->fraction = milliseconds - (milliseconds % fast_pow10[9 - digits]); + + *outbuf += sizeof(SQL_TIMESTAMP_STRUCT); + ind = sizeof(SQL_TIMESTAMP_STRUCT); + } + else if (pi->ValueType == SQL_C_CHAR && pi->BufferLength == TZAWARE_STRING_LENGTH) + { + char* p = _make_tzaware_string(cell); + if (!p) + return false; + memcpy(*outbuf, p, TZAWARE_STRING_LENGTH); + PyMem_Free(p); + *outbuf += TZAWARE_STRING_LENGTH; + ind = TZAWARE_STRING_LENGTH; + } + else return false; - SQL_TIMESTAMP_STRUCT *pts = (SQL_TIMESTAMP_STRUCT*)*outbuf; - pts->year = PyDateTime_GET_YEAR(cell); - pts->month = PyDateTime_GET_MONTH(cell); - pts->day = PyDateTime_GET_DAY(cell); - pts->hour = PyDateTime_DATE_GET_HOUR(cell); - pts->minute = PyDateTime_DATE_GET_MINUTE(cell); - pts->second = PyDateTime_DATE_GET_SECOND(cell); - - // Truncate the fraction according to precision - size_t digits = min(9, pi->DecimalDigits); - long fast_pow10[] = {1,10,100,1000,10000,100000,1000000,10000000,100000000,1000000000}; - SQLUINTEGER milliseconds = PyDateTime_DATE_GET_MICROSECOND(cell) * 1000; - pts->fraction = milliseconds - (milliseconds % fast_pow10[9 - digits]); - - *outbuf += sizeof(SQL_TIMESTAMP_STRUCT); - ind = sizeof(SQL_TIMESTAMP_STRUCT); } else if (PyDate_Check(cell)) { @@ -664,6 +785,25 @@ static bool GetBooleanInfo(Cursor* cur, Py_ssize_t index, PyObject* param, Param static bool GetDateTimeInfo(Cursor* cur, Py_ssize_t index, PyObject* param, ParamInfo& info) { + // If the value is not naïve, pass it as a string to preserve the timezone offset. + // See https://github.com/mkleehammer/pyodbc/issues/810. + if (cur->cnxn->preserve_tzoffsets && _datetime_is_tzaware(param)) + { + char* buf = _make_tzaware_string(param); + if (!buf) + return false; + info.ValueType = SQL_C_CHAR; + info.ParameterType = SQL_VARCHAR; + info.ColumnSize = (SQLULEN)TZAWARE_STRING_LENGTH; + info.DecimalDigits = 0; + info.ParameterValuePtr = buf; + info.BufferLength = (SQLLEN)TZAWARE_STRING_LENGTH; + info.StrLen_or_Ind = (SQLLEN)TZAWARE_STRING_LENGTH; + info.allocated = true; + return true; + } + + // Value is naïve, so proceed as before. info.Data.timestamp.year = (SQLSMALLINT) PyDateTime_GET_YEAR(param); info.Data.timestamp.month = (SQLUSMALLINT)PyDateTime_GET_MONTH(param); info.Data.timestamp.day = (SQLUSMALLINT)PyDateTime_GET_DAY(param); @@ -1707,7 +1847,7 @@ bool ExecuteMulti(Cursor* cur, PyObject* pSql, PyObject* paramArrayObj) Py_ssize_t i = 0; for (; i < cur->paramcount; i++) { - if (!DetectCType(cells[i], &cur->paramInfos[i])) + if (!DetectCType(cells[i], &cur->paramInfos[i], cur->cnxn->preserve_tzoffsets)) { goto ErrorRet3; } diff --git a/src/pyodbc.pyi b/src/pyodbc.pyi index cc236840..2caf88f9 100644 --- a/src/pyodbc.pyi +++ b/src/pyodbc.pyi @@ -398,6 +398,15 @@ class Connection: def maxwrite(self, value: int) -> None: ... + @property + def preserve_tzoffsets(self) -> bool: + """Opt-in flag to avoid discarding time zone offset with datetime parameters.""" + ... + + @preserve_tzoffsets.setter + def preserve_tzoffsets(self, value: bool) -> None: + ... + @property def readvar_initsize(self) -> int: """The initial buffer size in bytes for reading values from variable-length columns.""" diff --git a/tests/mysql_test.py b/tests/mysql_test.py index 732ae89c..103dcd1f 100644 --- a/tests/mysql_test.py +++ b/tests/mysql_test.py @@ -6,7 +6,7 @@ import ctypes import os from decimal import Decimal -from datetime import date, datetime +from datetime import date, datetime, timedelta, timezone from functools import lru_cache from typing import Iterator @@ -266,6 +266,49 @@ def test_datetime(cursor: pyodbc.Cursor): assert value == result +def test_datetime_with_offset(cursor: pyodbc.Cursor): + """Verify correct behavior of opt-in to preserve time zone offsets""" + tz = timezone(timedelta(hours=-12)) + aware = datetime(2007, 1, 15, 3, 4, 5, 123456, tzinfo=tz) + cursor.connection.autocommit = True + cursor.execute("create table t1 (dt timestamp(6))") + cursor.execute("create table t2 (id int, dt timestamp(6))") + + # Opt-in not set, offset should be ignored. + # Oracle's and MariaDB's ODBC drivers each have bugs here. + # https://bugs.mysql.com/bug.php?id=120402 + # https://jira.mariadb.org/browse/ODBC-492 + # Until those get fixed, we can't compare the entire object. + # naive = aware = datetime(2007, 1, 15, 3, 4, 5, 123456) + cursor.execute("insert into t1 values (?)", aware) + result = cursor.execute("select dt from t1").fetchval() + assert isinstance(result, datetime) + assert result.hour == 3 + + # Make sure the offset is applied to the result. The new opt-in behavior avoids those bugs. + cursor.execute("set time_zone = '+02:00'") + expected = datetime(2007, 1, 15, 17, 4, 5, 123456) + cursor.connection.preserve_tzoffsets = True + cursor.execute("insert into t2 values (?, ?)", (1, aware)) + result = cursor.execute("select dt from t2").fetchval() + assert result == expected + + # Try executemany. + params = (2, aware), (3, None), (4, aware), (5, aware) + cursor.executemany("insert into t2 values (?, ?)", params) + cursor.execute("select dt from t2 order by id") + results = [row[0] for row in cursor.fetchall()] + assert results == [expected, expected, None, expected, expected] + + # And for the pièce de résistance, fast_executemany. + cursor.fast_executemany = True + params = (6, aware), (7, None), (8, aware), (9, aware) + cursor.executemany("insert into t2 values (?, ?)", params) + cursor.execute("select dt from t2 order by id") + results = [row[0] for row in cursor.fetchall()] + assert results == [expected] * 2 + [None] + [expected] * 3 + [None] + [expected] * 2 + + def test_rowcount_delete(cursor: pyodbc.Cursor): cursor.execute("create table t1(i int)") count = 4 diff --git a/tests/postgresql_test.py b/tests/postgresql_test.py index d269b7aa..141c062f 100644 --- a/tests/postgresql_test.py +++ b/tests/postgresql_test.py @@ -5,6 +5,7 @@ import ctypes import os, uuid +from datetime import datetime, timedelta, timezone from decimal import Decimal from typing import Iterator @@ -645,6 +646,47 @@ def _test(): assert count_after == count_before +def test_datetime_with_offset(cursor: pyodbc.Cursor): + """Verify correct behavior of opt-in to preserve time zone offsets""" + tz = timezone(timedelta(hours=-5)) + naive = datetime(2007, 1, 15, 3, 4, 5, 987654) + aware = datetime(2007, 1, 15, 3, 4, 5, 987654, tzinfo=tz) + cursor.connection.autocommit = True + cursor.execute("create table t1 (dt timestamp)") + cursor.execute("create table t2 (id int, dt timestamptz)") + + # Opt-in not set, offset should be ignored + cursor.execute("insert into t1 values (?)", aware) + result = cursor.execute("select dt from t1").fetchval() + assert isinstance(result, datetime) + assert result == naive + + # Use a separate connection for reading the value. The cursor used for + # the INSERTs gets confused by SET TIME ZONE. + cursor2 = connect().cursor() + cursor2.execute("SET TIME ZONE 'Europe/Helsinki'") + + # Make sure the offset is preserved. + cursor.connection.preserve_tzoffsets = True + cursor.execute("insert into t2 values (?, ?)", (1, aware)) + result = cursor2.execute("select cast(dt AS varchar(50)) AS dt from t2").fetchval() + expected = "2007-01-15 10:04:05.987654+02" + assert result == expected + + # Try executemany. + cursor.executemany("insert into t2 values (?, ?)", [[2, aware], [3, None], [4, aware], [5, aware]]) + cursor2.execute("select cast(dt AS varchar(50)) AS dt from t2 order by id") + results = [row[0] for row in cursor2.fetchall()] + assert results == [expected, expected, None, expected, expected] + + # And for the curtain call, fast_executemany. + cursor.fast_executemany = True + cursor.executemany("insert into t2 values (?, ?)", [[6, aware], [7, None], [8, aware], [9, aware]]) + cursor2.execute("select cast(dt AS varchar(50)) AS dt from t2 order by id") + results = [row[0] for row in cursor2.fetchall()] + assert results == [expected, expected, None, expected, expected, expected, None, expected, expected] + + def test_rows_as_dicts(cursor: pyodbc.Cursor): """Test enhancement for ticket #171""" diff --git a/tests/sqlite_test.py b/tests/sqlite_test.py index f3bd94fc..e52dbfc8 100644 --- a/tests/sqlite_test.py +++ b/tests/sqlite_test.py @@ -23,7 +23,7 @@ import platform import re from collections.abc import Iterator -from datetime import datetime +from datetime import datetime, timedelta, timezone import pyodbc import pytest @@ -799,3 +799,33 @@ def test_rows_as_dicts(cursor: pyodbc.Cursor): row = cursor.execute("select name, name from t1").fetchone() assert len(row) == 1 assert row == {"name": "Kathleen"} + + +def test_datetime_with_offset(cursor: pyodbc.Cursor): + """Verify correct behavior of opt-in to preserve time zone offsets""" + tz = timezone(timedelta(hours=2)) + cursor.connection.autocommit = True + aware = datetime(2007, 1, 15, 3, 4, 5, 987654, tzinfo=tz) + cursor.execute("create table t1 (dt datetime)") + + # Opt-in not set, offset should be ignored + cursor.execute("insert into t1 values (?)", aware) + result = cursor.execute("select dt from t1").fetchval() + assert isinstance(result, datetime) + expected = datetime(2007, 1, 15, 3, 4, 5, 987000) + assert result == expected + + # Make sure the offset is preserved when the option is enabled. + cursor.execute("DELETE FROM t1") + cursor.connection.preserve_tzoffsets = True + cursor.execute("insert into t1 values (?)", aware) + result = cursor.execute("select cast(dt AS varchar(50)) AS dt from t1").fetchval() + expected = "2007-01-15 03:04:05.987654+02:00" + print(result) + assert result == expected + + # Try executemany (but not fast_executemany; see https://github.com/softace/sqliteodbc/issues/21). + cursor.executemany("insert into t1 values (?)", [[aware], [None], [aware], [aware]]) + cursor.execute("select cast(dt AS varchar(50)) AS dt from t1 order by 1") + results = [row[0] for row in cursor.fetchall()] + assert results == [None] + [expected] * 4 diff --git a/tests/sqlserver_test.py b/tests/sqlserver_test.py index 2a2e3eb6..8bfcc216 100755 --- a/tests/sqlserver_test.py +++ b/tests/sqlserver_test.py @@ -8,7 +8,7 @@ import uuid from collections.abc import Iterator from decimal import Decimal -from datetime import date, time, datetime +from datetime import date, time, datetime, timedelta, timezone from functools import lru_cache import pyodbc @@ -563,6 +563,46 @@ def test_datetime(cursor: pyodbc.Cursor): assert value == result +def test_datetime_with_offset(cursor: pyodbc.Cursor): + """Verify correct behavior of opt-in to preserve time zone offsets""" + tz = timezone(timedelta(hours=2)) + aware = datetime(2007, 1, 15, 3, 4, 5, 987654, tzinfo=tz) + cursor.execute("create table t1 (dt datetime)") + cursor.execute("create table t2 (dt datetimeoffset)") + cursor.commit() + + # Opt-in not set, offset should be ignored + cursor.execute("insert into t1 values (?)", aware) + result = cursor.execute("select dt from t1").fetchval() + assert isinstance(result, datetime) + expected = datetime(2007, 1, 15, 3, 4, 5, 987000) + assert result == expected + + # With the opt-in set, a exception should now be raised. + cursor.connection.preserve_tzoffsets = True + with pytest.raises(pyodbc.DataError): + cursor.execute("insert into t1 values (?)", aware) + + # Make sure the offset is preserved. + cursor.execute("insert into t2 values (?)", aware) + result = cursor.execute("select cast(dt AS varchar(50)) AS dt from t2").fetchval() + expected = "2007-01-15 03:04:05.9876540 +02:00" + assert result == expected + + # Try executemany. + cursor.executemany("insert into t2 values (?)", [[aware], [None], [aware], [aware]]) + cursor.execute("select cast(dt AS varchar(50)) AS dt from t2 order by 1") + results = [row[0] for row in cursor.fetchall()] + assert results == [None] + [expected] * 4 + + # And for the curtain call, fast_executemany. + cursor.fast_executemany = True + cursor.executemany("insert into t2 values (?)", [[aware], [None], [aware], [aware]]) + cursor.execute("select cast(dt AS varchar(50)) AS dt from t2 order by 1") + results = [row[0] for row in cursor.fetchall()] + assert results == [None, None] + [expected] * 7 + + def test_datetime_fraction(cursor: pyodbc.Cursor): # SQL Server supports milliseconds, but Python's datetime supports nanoseconds, so the most # granular datetime supported is xxx000.