From 1b427a851b90b82d2d402d66cd26f53bedc6377c Mon Sep 17 00:00:00 2001 From: Gus Monod Date: Mon, 20 Nov 2023 17:11:09 -0500 Subject: [PATCH 01/62] Drop support for 2.7 This commit contains the changes that `pyupgrade --py36-plus` recommends, plus the change of the `python_requires` to make it explicit that this version only supports 3.6+ Signed-off-by: Gus Monod --- comdb2/_cdb2_types.py | 16 +++--- comdb2/cdb2.py | 99 +++++++++++++++++------------------ comdb2/dbapi2.py | 17 +++--- comdb2/factories.py | 1 - docs/conf.py | 5 +- setup.py | 3 +- tests/test_cdb2.py | 7 ++- tests/test_cdb2_datetimeus.py | 1 - tests/test_dbapi2.py | 5 +- 9 files changed, 74 insertions(+), 80 deletions(-) diff --git a/comdb2/_cdb2_types.py b/comdb2/_cdb2_types.py index 908c344..e34af2f 100644 --- a/comdb2/_cdb2_types.py +++ b/comdb2/_cdb2_types.py @@ -36,11 +36,11 @@ class Error(RuntimeError): after the failed call. """ def __init__(self, error_code, error_message): - if not(isinstance(error_message, six.text_type)): + if not(isinstance(error_message, str)): error_message = _errstr(error_message) self.error_code = error_code self.error_message = error_message - super(Error, self).__init__(error_code, error_message) + super().__init__(error_code, error_message) class Effects(namedtuple('Effects', @@ -94,13 +94,13 @@ def fromdatetime(cls, dt): dt.tzinfo, **kwargs) def __add__(self, other): - ret = super(DatetimeUs, self).__add__(other) + ret = super().__add__(other) if isinstance(ret, datetime): return DatetimeUs.fromdatetime(ret) return ret # must be a timedelta def __sub__(self, other): - ret = super(DatetimeUs, self).__sub__(other) + ret = super().__sub__(other) if isinstance(ret, datetime): return DatetimeUs.fromdatetime(ret) return ret # must be a timedelta @@ -110,20 +110,20 @@ def __radd__(self, other): @classmethod def now(cls, tz=None): - ret = super(DatetimeUs, cls).now(tz) + ret = super().now(tz) return DatetimeUs.fromdatetime(ret) @classmethod def fromtimestamp(cls, timestamp, tz=None): - ret = super(DatetimeUs, cls).fromtimestamp(timestamp, tz) + ret = super().fromtimestamp(timestamp, tz) return DatetimeUs.fromdatetime(ret) def astimezone(self, *args, **kwargs): - ret = super(DatetimeUs, self).astimezone(*args, **kwargs) + ret = super().astimezone(*args, **kwargs) return DatetimeUs.fromdatetime(ret) def replace(self, *args, **kwargs): # Before Python 3.7, it is effectively implementation dependent whether # this returns a DatetimeUs or a datetime. - dt = super(DatetimeUs, self).replace(*args, **kwargs) + dt = super().replace(*args, **kwargs) return self.fromdatetime(dt) diff --git a/comdb2/cdb2.py b/comdb2/cdb2.py index 37a67dd..e38a2c5 100644 --- a/comdb2/cdb2.py +++ b/comdb2/cdb2.py @@ -127,7 +127,6 @@ problems for new users. Make sure to carefully read :ref:`String and Blob Types` on that page. """ -from __future__ import absolute_import, unicode_literals from ._cdb2_types import Error, Effects, DatetimeUs from ._ccdb2 import Handle as CHandle @@ -136,40 +135,40 @@ 'ERROR_CODE', 'TYPE', 'HANDLE_FLAGS'] # Pull all comdb2 error codes from cdb2api.h into our namespace -ERROR_CODE = {u'CONNECT_ERROR' : -1, - u'NOTCONNECTED' : -2, - u'PREPARE_ERROR' : -3, - u'IO_ERROR' : -4, - u'INTERNAL' : -5, - u'NOSTATEMENT' : -6, - u'BADCOLUMN' : -7, - u'BADSTATE' : -8, - u'ASYNCERR' : -9, - u'INVALID_ID' : -12, - u'RECORD_OUT_OF_RANGE' : -13, - u'REJECTED' : -15, - u'STOPPED' : -16, - u'BADREQ' : -17, - u'DBCREATE_FAILED' : -18, - u'THREADPOOL_INTERNAL' : -20, - u'READONLY' : -21, - u'NOMASTER' : -101, - u'UNTAGGED_DATABASE' : -102, - u'CONSTRAINTS' : -103, - u'DEADLOCK' : 203, - u'TRAN_IO_ERROR' : -105, - u'ACCESS' : -106, - u'TRAN_MODE_UNSUPPORTED' : -107, - u'VERIFY_ERROR' : 2, - u'FKEY_VIOLATION' : 3, - u'NULL_CONSTRAINT' : 4, - u'CONV_FAIL' : 113, - u'NONKLESS' : 114, - u'MALLOC' : 115, - u'NOTSUPPORTED' : 116, - u'DUPLICATE' : 299, - u'TZNAME_FAIL' : 401, - u'UNKNOWN' : 300, +ERROR_CODE = {'CONNECT_ERROR' : -1, + 'NOTCONNECTED' : -2, + 'PREPARE_ERROR' : -3, + 'IO_ERROR' : -4, + 'INTERNAL' : -5, + 'NOSTATEMENT' : -6, + 'BADCOLUMN' : -7, + 'BADSTATE' : -8, + 'ASYNCERR' : -9, + 'INVALID_ID' : -12, + 'RECORD_OUT_OF_RANGE' : -13, + 'REJECTED' : -15, + 'STOPPED' : -16, + 'BADREQ' : -17, + 'DBCREATE_FAILED' : -18, + 'THREADPOOL_INTERNAL' : -20, + 'READONLY' : -21, + 'NOMASTER' : -101, + 'UNTAGGED_DATABASE' : -102, + 'CONSTRAINTS' : -103, + 'DEADLOCK' : 203, + 'TRAN_IO_ERROR' : -105, + 'ACCESS' : -106, + 'TRAN_MODE_UNSUPPORTED' : -107, + 'VERIFY_ERROR' : 2, + 'FKEY_VIOLATION' : 3, + 'NULL_CONSTRAINT' : 4, + 'CONV_FAIL' : 113, + 'NONKLESS' : 114, + 'MALLOC' : 115, + 'NOTSUPPORTED' : 116, + 'DUPLICATE' : 299, + 'TZNAME_FAIL' : 401, + 'UNKNOWN' : 300, } """This dict maps all known Comdb2 error names to their respective values. @@ -180,15 +179,15 @@ """ # Pull comdb2 column types from cdb2api.h into our namespace -TYPE = {u'INTEGER' : 1, - u'REAL' : 2, - u'CSTRING' : 3, - u'BLOB' : 4, - u'DATETIME' : 6, - u'INTERVALYM' : 7, - u'INTERVALDS' : 8, - u'DATETIMEUS' : 9, - u'INTERVALDSUS' : 10, +TYPE = {'INTEGER' : 1, + 'REAL' : 2, + 'CSTRING' : 3, + 'BLOB' : 4, + 'DATETIME' : 6, + 'INTERVALYM' : 7, + 'INTERVALDS' : 8, + 'DATETIMEUS' : 9, + 'INTERVALDSUS' : 10, } """This dict maps all known Comdb2 types to their enumeration value. @@ -198,11 +197,11 @@ """ # Pull comdb2 handle flags from cdb2api.h into our namespace -HANDLE_FLAGS = {u'READ_INTRANS_RESULTS' : 2, - u'DIRECT_CPU' : 4, - u'RANDOM' : 8, - u'RANDOMROOM' : 16, - u'ROOM' : 32, +HANDLE_FLAGS = {'READ_INTRANS_RESULTS' : 2, + 'DIRECT_CPU' : 4, + 'RANDOM' : 8, + 'RANDOMROOM' : 16, + 'ROOM' : 32, } """This dict maps all known Comdb2 flags to their enumeration value. @@ -211,7 +210,7 @@ """ -class Handle(object): +class Handle: """Represents a connection to a database. By default, the connection will be made to the cluster configured as the diff --git a/comdb2/dbapi2.py b/comdb2/dbapi2.py index cacf68c..6c9e0b1 100644 --- a/comdb2/dbapi2.py +++ b/comdb2/dbapi2.py @@ -239,7 +239,6 @@ `Connection.cursor` is called, we call `Cursor.close` on any existing, open cursor for that connection. """ -from __future__ import absolute_import, unicode_literals import functools import itertools @@ -304,7 +303,7 @@ @functools.total_ordering -class _TypeObject(object): +class _TypeObject: def __init__(self, *value_names): self.value_names = value_names self.values = [cdb2.TYPE[v] for v in value_names] @@ -320,7 +319,7 @@ def __repr__(self): def _binary(string): - if isinstance(string, six.text_type): + if isinstance(string, str): return string.encode('utf-8') return bytes(string) @@ -526,8 +525,8 @@ def _raise_wrapped_exception(exc): code = exc.error_code msg = '%s (cdb2api rc %d)' % (exc.error_message, code) if "null constraint violation" in msg: - six.raise_from(NonNullConstraintError(msg), exc) # DRQS 86013831 - six.raise_from(_EXCEPTION_BY_RC.get(code, OperationalError)(msg), exc) + raise NonNullConstraintError(msg) from exc # DRQS 86013831 + raise _EXCEPTION_BY_RC.get(code, OperationalError)(msg) from exc def _sql_operation(sql): @@ -566,7 +565,7 @@ def connect(*args, **kwargs): return Connection(*args, **kwargs) -class Connection(object): +class Connection: """Represents a connection to a Comdb2 database. By default, the connection will be made to the cluster configured as the @@ -774,7 +773,7 @@ def cursor(self): NotSupportedError = NotSupportedError -class Cursor(object): +class Cursor: """Class used to send requests through a database connection. This class is not meant to be instantiated directly; it should always be @@ -1028,10 +1027,10 @@ def _execute(self, operation, sql, parameters=None): sql = sql % {name: "@" + name for name in parameters} except KeyError as keyerr: msg = "No value provided for parameter %s" % keyerr - six.raise_from(InterfaceError(msg), keyerr) + raise InterfaceError(msg) from keyerr except Exception as exc: msg = "Invalid Python format string for query" - six.raise_from(InterfaceError(msg), exc) + raise InterfaceError(msg) from exc if _operation_ends_transaction(operation): self._conn._in_transaction = False # txn ends, even on failure diff --git a/comdb2/factories.py b/comdb2/factories.py index c63e5f8..d518aae 100644 --- a/comdb2/factories.py +++ b/comdb2/factories.py @@ -21,7 +21,6 @@ A factory function will be called with a list of column names, and must return a callable that will be called once per row with a list of column values. """ -from __future__ import unicode_literals from collections import namedtuple from collections import Counter diff --git a/docs/conf.py b/docs/conf.py index c6c59ec..d996b14 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Comdb2 documentation build configuration file, created by # sphinx-quickstart on Thu Sep 8 16:16:10 2016. @@ -47,8 +46,8 @@ root_doc = 'index' # General information about the project. -project = u'Comdb2' -copyright = u'2017, Bloomberg LP' +project = 'Comdb2' +copyright = '2017, Bloomberg LP' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/setup.py b/setup.py index ad78159..1bf8401 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,8 @@ def make_static_extension(name, **kwargs): author_email='achamberlai9@bloomberg.net', packages=['comdb2'], install_requires=["six", "pytz"], - extras_require={"tests": ["python-dateutil>=2.6.0", "pytest", "mock; python_version < '3.3'"]}, + extras_require={"tests": ["python-dateutil>=2.6.0", "pytest"]}, + python_requires=">=3.6", ext_modules=[ccdb2], package_data={"comdb2": ["py.typed", "*.pyi"]}, ) diff --git a/tests/test_cdb2.py b/tests/test_cdb2.py index 6620baa..7c24bce 100644 --- a/tests/test_cdb2.py +++ b/tests/test_cdb2.py @@ -9,7 +9,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import unicode_literals, absolute_import from comdb2 import cdb2 from comdb2.factories import dict_row_factory @@ -153,12 +152,12 @@ def test_row_factories_with_dup_col_names(): hndl.row_factory = namedtuple_row_factory with pytest.raises(cdb2.Error) as exc_info: hndl.execute(query) - assert isinstance(exc_info.value.args[1], six.text_type) + assert isinstance(exc_info.value.args[1], str) hndl.row_factory = dict_row_factory with pytest.raises(cdb2.Error) as exc_info: hndl.execute(query) - assert isinstance(exc_info.value.args[1], six.text_type) + assert isinstance(exc_info.value.args[1], str) def test_setting_non_callable_row_factory(): @@ -193,7 +192,7 @@ def klass(col_values): it = iter(hndl.execute(query)) with pytest.raises(cdb2.Error) as exc_info: next(it) - assert isinstance(exc_info.value.args[1], six.text_type) + assert isinstance(exc_info.value.args[1], str) def test_get_effects(): diff --git a/tests/test_cdb2_datetimeus.py b/tests/test_cdb2_datetimeus.py index d1a9462..36601d6 100644 --- a/tests/test_cdb2_datetimeus.py +++ b/tests/test_cdb2_datetimeus.py @@ -9,7 +9,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import unicode_literals, absolute_import from comdb2 import cdb2 import pytz diff --git a/tests/test_dbapi2.py b/tests/test_dbapi2.py index 26344f5..0a8d8bf 100644 --- a/tests/test_dbapi2.py +++ b/tests/test_dbapi2.py @@ -9,7 +9,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import unicode_literals, absolute_import from comdb2.dbapi2 import _sql_operation from comdb2.dbapi2 import NUMBER @@ -43,7 +42,7 @@ try: from unittest.mock import patch except ImportError: - from mock import patch + from unittest.mock import patch COLUMN_LIST = ("short_col u_short_col int_col u_int_col longlong_col" " float_col double_col byte_col byte_array_col" @@ -618,7 +617,7 @@ def test_exceptions_containing_unicode_error_messages(): try: cursor.execute("select") except ProgrammingError as exc: - assert isinstance(exc.args[0], six.text_type) + assert isinstance(exc.args[0], str) raise From e100fa1f85a917a6ff85a7346e51b1b797ace5d9 Mon Sep 17 00:00:00 2001 From: Gus Monod Date: Mon, 20 Nov 2023 17:11:59 -0500 Subject: [PATCH 02/62] Bump minor version This minor version is the first version that removes support for Python 2.7. Signed-off-by: Gus Monod --- comdb2/_about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comdb2/_about.py b/comdb2/_about.py index 09033dd..3dfc66f 100644 --- a/comdb2/_about.py +++ b/comdb2/_about.py @@ -1,2 +1,2 @@ __all__ = ['__version__'] -__version__ = "1.4.1" +__version__ = "1.5.0" From dae3858715611ae6a0df6f0ea0a6d0845ab4958c Mon Sep 17 00:00:00 2001 From: Gus Monod Date: Mon, 20 Nov 2023 17:44:02 -0500 Subject: [PATCH 03/62] Remove `six` compatibility library This library is necessary only for Python 2 support, and now that we have dropped support for 2.7, we don't need it anymore. Signed-off-by: Gus Monod --- comdb2/__init__.py | 2 +- comdb2/_ccdb2.pyx | 11 +++++------ comdb2/_cdb2_types.py | 1 - comdb2/cdb2.py | 4 ++-- comdb2/dbapi2.py | 12 ++++------- docs/conf.py | 3 +-- docs/dbapi2.rst | 6 +++--- docs/tips.rst | 14 +++++-------- docs/types.rst | 46 +++++++++++-------------------------------- setup.py | 2 +- tests/test_cdb2.py | 1 - tests/test_dbapi2.py | 1 - 12 files changed, 33 insertions(+), 70 deletions(-) diff --git a/comdb2/__init__.py b/comdb2/__init__.py index da5f68f..14d579e 100644 --- a/comdb2/__init__.py +++ b/comdb2/__init__.py @@ -12,7 +12,7 @@ """This package provides Python interfaces to Comdb2 databases. Two different Python submodules are provided for interacting with Comdb2 -databases. Both submodules work from Python 2.7+ and from Python 3.5+. +databases. Both submodules work from Python 3.6. `comdb2.dbapi2` provides an interface that conforms to `the Python Database API Specification v2.0 `_. If you're diff --git a/comdb2/_ccdb2.pyx b/comdb2/_ccdb2.pyx index a439d91..155bacb 100644 --- a/comdb2/_ccdb2.pyx +++ b/comdb2/_ccdb2.pyx @@ -22,7 +22,6 @@ from cpython.mem cimport PyMem_Malloc, PyMem_Free import datetime from pytz import timezone, UTC -import six from ._cdb2_types import Error, Effects, DatetimeUs from . cimport _cdb2api as lib @@ -113,7 +112,7 @@ cdef class _ParameterValue(object): self.size = 0 self.data = NULL return - elif isinstance(obj, six.integer_types): + elif isinstance(obj, int): self.type = lib.CDB2_INTEGER self.owner = None self.size = sizeof(long long) @@ -167,7 +166,7 @@ cdef class _ParameterValue(object): param_name, exc_desc, ) - six.raise_from(Error(lib.CDB2ERR_CONV_FAIL, errmsg), exc) + raise Error(lib.CDB2ERR_CONV_FAIL, errmsg) from exc else: errmsg = "Can't map %s value %r for parameter '%s' to a Comdb2 type" % ( type(obj).__name__, @@ -248,7 +247,7 @@ cdef _column_value(lib.cdb2_hndl_tp *hndl, int col): if exc is not None: errmsg += _describe_exception(exc) - six.raise_from(Error(lib.CDB2ERR_CONV_FAIL, errmsg), exc) + raise Error(lib.CDB2ERR_CONV_FAIL, errmsg) from exc else: errmsg += "Unsupported column type" raise Error(lib.CDB2ERR_NOTSUPPORTED, errmsg) @@ -289,7 +288,7 @@ cdef class _Cursor(object): ret = self.row_class(ret) except Exception as e: errmsg = "Instantiating row failed: " + _describe_exception(e) - six.raise_from(Error(lib.CDB2ERR_UNKNOWN, errmsg), e) + raise Error(lib.CDB2ERR_UNKNOWN, errmsg) from e return ret @@ -393,7 +392,7 @@ cdef class Handle(object): row_class = self._row_factory(self.column_names()) except Exception as e: errmsg = "row_factory call failed: " + _describe_exception(e) - six.raise_from(Error(lib.CDB2ERR_UNKNOWN, errmsg), e) + raise Error(lib.CDB2ERR_UNKNOWN, errmsg) from e cdef _Cursor ret = _Cursor.__new__(_Cursor) ret.handle = self diff --git a/comdb2/_cdb2_types.py b/comdb2/_cdb2_types.py index e34af2f..004febb 100644 --- a/comdb2/_cdb2_types.py +++ b/comdb2/_cdb2_types.py @@ -11,7 +11,6 @@ from collections import namedtuple from datetime import datetime -import six __name__ = 'comdb2.cdb2' diff --git a/comdb2/cdb2.py b/comdb2/cdb2.py index e38a2c5..4ecd51a 100644 --- a/comdb2/cdb2.py +++ b/comdb2/cdb2.py @@ -112,8 +112,8 @@ NULL ``None`` integer `int` real `float` -blob `six.binary_type` (aka `bytes` in Python 3, ``str`` in Python 2) -text `six.text_type` (aka `str` in Python 3, ``unicode`` in Python 2) +blob `bytes` +text `str` datetime `datetime.datetime` datetimeus `DatetimeUs` ============ ================================================================ diff --git a/comdb2/dbapi2.py b/comdb2/dbapi2.py index 6c9e0b1..3aba62e 100644 --- a/comdb2/dbapi2.py +++ b/comdb2/dbapi2.py @@ -138,8 +138,8 @@ NULL ``None`` integer `int` real `float` -blob `six.binary_type` (aka `bytes` in Python 3, ``str`` in Python 2) -text `six.text_type` (aka `str` in Python 3, ``unicode`` in Python 2) +blob `bytes` +text `str` datetime `datetime.datetime` datetimeus `DatetimeUs` ============ ================================================================ @@ -245,7 +245,6 @@ import weakref import datetime import re -import six from . import cdb2 @@ -297,7 +296,7 @@ \s* # then skip over any whitespace (\w+) # and capture the first word """, - re.VERBOSE | re.DOTALL | (0 if six.PY2 else re.ASCII), + re.VERBOSE | re.DOTALL | re.ASCII, ) _VALID_SP_NAME = re.compile(r'^[A-Za-z0-9_.]+$') @@ -349,10 +348,7 @@ def _binary(string): TimestampFromTicks = Timestamp.fromtimestamp TimestampUsFromTicks = TimestampUs.fromtimestamp -try: - UserException = StandardError # Python 2 -except NameError: - UserException = Exception # Python 3 +UserException = Exception class Error(UserException): diff --git a/docs/conf.py b/docs/conf.py index d996b14..8e2f22b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -261,8 +261,7 @@ # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'python': ('https://docs.python.org/3', None), - 'six': ('https://six.readthedocs.io', None)} +intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} # Don't highlight the code for the Python standard library datetime module. viewcode_import = False diff --git a/docs/dbapi2.rst b/docs/dbapi2.rst index 9ad9692..4030a90 100644 --- a/docs/dbapi2.rst +++ b/docs/dbapi2.rst @@ -87,9 +87,9 @@ them if you so choose, but you are not required to. Creates an object suitable for binding as a BLOB parameter. - If the input argument was a `six.text_type` Unicode string, it is - encoded as a UTF-8 byte string and returned. Otherwise, the input - argument is passed to the `bytes` constructor, and the result returned. + If the input argument was a `str` object, it is encoded as a UTF-8 byte + string and returned. Otherwise, the input argument is passed to the + `bytes` constructor, and the result returned. :param string: A string from which the new object is constructed :rtype: `bytes` diff --git a/docs/tips.rst b/docs/tips.rst index e5c9fdf..c79968e 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -2,10 +2,9 @@ Best Practices, Tips, and Tricks ******************************** -#. This package uses `six.text_type` for text and `bytes` for blobs; this may - be surprising if you're using `str` in Python 2. See :doc:`types` for an - explanation of what Python types must be used when binding parameters, and - what Python types will be returned for result columns. +#. This package uses `str` for text and `bytes` for blobs. See :doc:`types` for + an explanation of what Python types must be used when binding parameters, + and what Python types will be returned for result columns. An error message saying "incompatible values from SQL blob of length 3 to bcstring field 'foo'" most likely means that you passed a byte string where @@ -24,11 +23,8 @@ Best Practices, Tips, and Tricks a ``WHERE`` clause that you expected to match some rows will silently fail to match any. -#. Prefer using Python 3 rather than Python 2 when possible. The above type - mappings are much more intuitive in Python 3 than in Python 2. If you can't - use Python 3 but are writing a new module, prefer to use ``from __future__ - import unicode_literals`` to opt into forward compatible unicode string - literals (rather than the default, string literals as byte strings). +#. The latest version of this package only supports Python 3. If you can't + use Python 3, make sure to use version 1.4.1 or earlier. #. The database can time out connections that have been idle for a period of time, and each idle connection uses some amount of resources on the database diff --git a/docs/types.rst b/docs/types.rst index 0fd1da9..788dd6d 100644 --- a/docs/types.rst +++ b/docs/types.rst @@ -14,8 +14,8 @@ SQL type Python type NULL ``None`` integer `int` real `float` -blob `six.binary_type` (aka `bytes` in Python 3, ``str`` in Python 2) -text `six.text_type` (aka `str` in Python 3, ``unicode`` in Python 2) +blob `bytes` +text `str` datetime `datetime.datetime` datetimeus `~.cdb2.DatetimeUs` intervalym not supported @@ -47,43 +47,19 @@ represent TEXT columns. This was chosen for maximum forward compatibility with Python 3, and to make it easier to write code that will work identically in both languages. This decision has many important ramifications. -#. In Python 2, ``'foo'`` is the wrong type for binding a ``cstring`` column. - You instead need to use a Unicode literal like ``u'foo'``. - Alternately, you can make all string literals in your module be Unicode - literals by default by doing:: +#. If you have a variable of type `bytes` and you want to pass it to the + database as a TEXT value, you need to convert it to a `str` using + `~bytes.decode`. - from __future__ import unicode_literals - - .. note:: - It's not safe to blindly do this in existing code as it changes that - code's behavior, and might cause problems. - -#. If you have a variable of type `six.binary_type` (byte string) and you - want to pass it to the database as a TEXT value, you need to convert it to - a `six.text_type` (Unicode) string using `~bytes.decode`. In Python 2, - unless ``unicode_literals`` was imported from `__future__`, you will need to - do this with every string variable that you want to use as a cstring - column. For example:: - - value = 'foo' - params = {'col': value.decode('utf-8')} - # dbapi2 - cursor.execute("select * from tbl where col=%(col)s", params) - # cdb2 - handle.execute("select * from tbl where col=@col", params) - -#. TEXT columns are returned to you as `six.text_type` (Unicode) strings. - If you need to pass them to a library that expects `six.binary_type` (byte) - strings, you have to use `~str.encode`, like this:: +#. TEXT columns are returned to you as `str` strings. If you need to pass them + to a library that expects `bytes` strings, you have to use `~str.encode`, + like this:: cursor.execute("select col from tbl") for row in cursor: col = row[0] library_func(col.encode('utf-8')) - This may come up a lot in Python 2, where historically libraries were - written to expect byte strings rather than Unicode strings. - #. When a TEXT column is read back from the database, it is decoded as UTF-8. If your database has non-UTF-8 text in a ``cstring`` column, you need to cast that column from a TEXT to a BLOB in SQL when reading it out, like @@ -91,14 +67,14 @@ both languages. This decision has many important ramifications. cursor.execute("select cast(col as blob) from tbl") - That will result in you receiving a byte string rather than a Unicode - string for that result column. + That will result in you receiving a `bytes` object rather than `str` for + that result column. .. note:: ASCII is a subset of UTF-8, so this is not required for if the column contains plain ASCII text. -#. Likewise, when a Unicode string is sent to the database, it is encoded as +#. Likewise, when a `str` object is sent to the database, it is encoded as UTF-8. If you need to store non-UTF-8 text in a ``cstring`` column in your database, you need to bind a byte string in Python land, and then cast the value from BLOB to TEXT in SQL. For instance, to bind a latin1 string diff --git a/setup.py b/setup.py index 1bf8401..ff100f6 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ def make_static_extension(name, **kwargs): author='Alex Chamberlain', author_email='achamberlai9@bloomberg.net', packages=['comdb2'], - install_requires=["six", "pytz"], + install_requires=["pytz"], extras_require={"tests": ["python-dateutil>=2.6.0", "pytest"]}, python_requires=">=3.6", ext_modules=[ccdb2], diff --git a/tests/test_cdb2.py b/tests/test_cdb2.py index 7c24bce..381bc9a 100644 --- a/tests/test_cdb2.py +++ b/tests/test_cdb2.py @@ -14,7 +14,6 @@ from comdb2.factories import dict_row_factory from comdb2.factories import namedtuple_row_factory import pytest -import six COLUMN_LIST = ("short_col u_short_col int_col u_int_col longlong_col" " float_col double_col byte_col byte_array_col" diff --git a/tests/test_dbapi2.py b/tests/test_dbapi2.py index 0a8d8bf..339ce79 100644 --- a/tests/test_dbapi2.py +++ b/tests/test_dbapi2.py @@ -36,7 +36,6 @@ import pytest import datetime import pytz -import six from functools import partial try: From 5caf25d4d1c6846d1a66945bf1988bf886c0862b Mon Sep 17 00:00:00 2001 From: Gus Monod Date: Mon, 20 Nov 2023 17:47:48 -0500 Subject: [PATCH 04/62] Remove unnecessary try/except This was made unnecessary by the running of `pyupgrade --py36-plus` Signed-off-by: Gus Monod --- tests/test_dbapi2.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_dbapi2.py b/tests/test_dbapi2.py index 339ce79..40e33fa 100644 --- a/tests/test_dbapi2.py +++ b/tests/test_dbapi2.py @@ -38,10 +38,7 @@ import pytz from functools import partial -try: - from unittest.mock import patch -except ImportError: - from unittest.mock import patch +from unittest.mock import patch COLUMN_LIST = ("short_col u_short_col int_col u_int_col longlong_col" " float_col double_col byte_col byte_array_col" From 5a0e834c27e157a4b0a5c251907edeae048d5425 Mon Sep 17 00:00:00 2001 From: Gus Monod Date: Mon, 20 Nov 2023 18:38:14 -0500 Subject: [PATCH 05/62] Reword some of the docs to be focused on Python 3 Signed-off-by: Gus Monod --- docs/tips.rst | 2 +- docs/types.rst | 14 +------------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/docs/tips.rst b/docs/tips.rst index c79968e..5dc9afd 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -24,7 +24,7 @@ Best Practices, Tips, and Tricks to match any. #. The latest version of this package only supports Python 3. If you can't - use Python 3, make sure to use version 1.4.1 or earlier. + use Python 3, make sure to use version less than ``1.5.0``. #. The database can time out connections that have been idle for a period of time, and each idle connection uses some amount of resources on the database diff --git a/docs/types.rst b/docs/types.rst index 788dd6d..79bfaf5 100644 --- a/docs/types.rst +++ b/docs/types.rst @@ -42,24 +42,12 @@ into a column of type ``short``), an exception will be raised. String and Blob Types ===================== -This module uses byte strings to represent BLOB values, and Unicode strings to -represent TEXT columns. This was chosen for maximum forward compatibility with -Python 3, and to make it easier to write code that will work identically in -both languages. This decision has many important ramifications. +You may occasionally need to convert between byte strings and Unicode strings: #. If you have a variable of type `bytes` and you want to pass it to the database as a TEXT value, you need to convert it to a `str` using `~bytes.decode`. -#. TEXT columns are returned to you as `str` strings. If you need to pass them - to a library that expects `bytes` strings, you have to use `~str.encode`, - like this:: - - cursor.execute("select col from tbl") - for row in cursor: - col = row[0] - library_func(col.encode('utf-8')) - #. When a TEXT column is read back from the database, it is decoded as UTF-8. If your database has non-UTF-8 text in a ``cstring`` column, you need to cast that column from a TEXT to a BLOB in SQL when reading it out, like From 7b81430f0d8701ddb0d86d099980e40455c6a231 Mon Sep 17 00:00:00 2001 From: Gus Monod Date: Tue, 21 Nov 2023 11:52:15 -0500 Subject: [PATCH 06/62] Drop support for 3.6 Change `python_requires` to make it explicit that this version only supports 3.7+ and bump the minor version. Note that `pyupgrade --py37-plus` is a no-op. Signed-off-by: Gus Monod --- comdb2/__init__.py | 2 +- comdb2/_about.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/comdb2/__init__.py b/comdb2/__init__.py index 14d579e..2de7db9 100644 --- a/comdb2/__init__.py +++ b/comdb2/__init__.py @@ -12,7 +12,7 @@ """This package provides Python interfaces to Comdb2 databases. Two different Python submodules are provided for interacting with Comdb2 -databases. Both submodules work from Python 3.6. +databases. `comdb2.dbapi2` provides an interface that conforms to `the Python Database API Specification v2.0 `_. If you're diff --git a/comdb2/_about.py b/comdb2/_about.py index 3dfc66f..1180e5a 100644 --- a/comdb2/_about.py +++ b/comdb2/_about.py @@ -1,2 +1,2 @@ __all__ = ['__version__'] -__version__ = "1.5.0" +__version__ = "1.6.0" diff --git a/setup.py b/setup.py index ff100f6..63d9b94 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ def make_static_extension(name, **kwargs): packages=['comdb2'], install_requires=["pytz"], extras_require={"tests": ["python-dateutil>=2.6.0", "pytest"]}, - python_requires=">=3.6", + python_requires=">=3.7", ext_modules=[ccdb2], package_data={"comdb2": ["py.typed", "*.pyi"]}, ) From 467128c539a90973156f3373ed7a97d099637919 Mon Sep 17 00:00:00 2001 From: asardesai2 Date: Thu, 9 Nov 2023 20:46:10 +0530 Subject: [PATCH 07/62] Allow parameter binding Arrays and Tuples Signed-off-by: asardesai2 --- comdb2/_ccdb2.pyx | 67 +++++++++++++++++++++++++++++++++++++++++--- comdb2/_cdb2api.pxd | 3 +- tests/test_dbapi2.py | 64 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 5 deletions(-) diff --git a/comdb2/_ccdb2.pyx b/comdb2/_ccdb2.pyx index 155bacb..bcaca37 100644 --- a/comdb2/_ccdb2.pyx +++ b/comdb2/_ccdb2.pyx @@ -39,6 +39,11 @@ ctypedef fused client_datetime: lib.cdb2_client_datetimeus_t +cdef struct blob_descriptor: + size_t size + char* data + + cdef _string_as_bytes(s): if isinstance(s, unicode): return s.encode('utf-8') @@ -99,13 +104,15 @@ cdef _bind_datetime(obj, client_datetime *val): cdef class _ParameterValue(object): - cdef int type + cdef lib.cdb2_coltype type cdef int size cdef void *data cdef object owner + cdef int list_size def __cinit__(self, obj, param_name): try: + self.list_size = -1 if obj is None: self.type = lib.CDB2_INTEGER self.owner = None @@ -152,6 +159,54 @@ cdef class _ParameterValue(object): self.data = PyMem_Malloc(self.size) _bind_datetime(obj, self.data) return + elif isinstance(obj, (list, tuple)): + self.list_size = len(obj) + if 0 == self.list_size: + raise ValueError(f"empty {type(obj).__name__}s cannot be bound") + + if all(isinstance(ele, int) for ele in obj): + self.type = lib.CDB2_INTEGER + self.size = sizeof(long long) + self.owner = None + self.data = PyMem_Malloc(self.list_size * self.size) + for l_index in range(self.list_size): + (self.data)[l_index] = obj[l_index] + return + elif all(isinstance(ele, float) for ele in obj): + self.type = lib.CDB2_REAL + self.size = sizeof(double) + self.owner = None + self.data = PyMem_Malloc(self.list_size * self.size) + for l_index in range(self.list_size): + (self.data)[l_index] = obj[l_index] + return + elif all(isinstance(ele, bytes) for ele in obj): + self.type = lib.CDB2_BLOB + self.size = sizeof(blob_descriptor) + self.owner = obj + self.data = PyMem_Malloc(self.list_size * self.size) + for l_index in range(self.list_size): + (self.data)[l_index].size = len(obj[l_index]) + (self.data)[l_index].data = obj[l_index] + return + elif all(isinstance(ele, unicode) for ele in obj): + self.type = lib.CDB2_CSTRING + self.size = sizeof(char*) + # Strings need to be converted to bytes + self.owner = [x.encode('utf-8') for x in obj] + self.data = PyMem_Malloc(self.list_size * self.size) + for l_index in range(self.list_size): + (self.data)[l_index] = (self.owner[l_index]) + return + elif not all(isinstance(ele, type(obj[0])) for ele in obj): + raise ValueError( + f"all {type(obj).__name__} elements must be the same type" + ) + else: + raise ValueError( + f"Cannot bind a {type(obj).__name__} of {type(obj[0]).__name__}" + ) + except Exception as e: exc = e else: @@ -177,7 +232,7 @@ cdef class _ParameterValue(object): def __dealloc__(self): - if self.owner is None: + if self.owner is None or self.list_size != -1: PyMem_Free(self.data) @@ -343,8 +398,12 @@ cdef class Handle(object): cval = _ParameterValue(val, key) param_guards.append(ckey) param_guards.append(cval) - rc = lib.cdb2_bind_param(self.hndl, ckey, - cval.type, cval.data, cval.size) + if cval.list_size == -1: + rc = lib.cdb2_bind_param(self.hndl, ckey, + cval.type, cval.data, cval.size) + else: + # Bind Array if cval is an array + rc = lib.cdb2_bind_array(self.hndl, ckey, cval.type, cval.data, cval.list_size, cval.size) _errchk(rc, self.hndl) with nogil: diff --git a/comdb2/_cdb2api.pxd b/comdb2/_cdb2api.pxd index 40c84ab..afae380 100644 --- a/comdb2/_cdb2api.pxd +++ b/comdb2/_cdb2api.pxd @@ -18,7 +18,7 @@ cdef extern from "cdb2api.h" nogil: CDB2ERR_NOTSUPPORTED CDB2ERR_CONV_FAIL - enum: + enum cdb2_coltype: CDB2_INTEGER CDB2_REAL CDB2_CSTRING @@ -75,5 +75,6 @@ cdef extern from "cdb2api.h" nogil: void* cdb2_column_value(cdb2_hndl_tp* hndl, int col); const char* cdb2_errstr(cdb2_hndl_tp* hndl); int cdb2_bind_param(cdb2_hndl_tp *hndl, const char *name, int type, const void *varaddr, int length); + int cdb2_bind_array(cdb2_hndl_tp *hndl, const char *name, cdb2_coltype, const void *varaddr, size_t count, size_t typelen); int cdb2_clearbindings(cdb2_hndl_tp *hndl); int cdb2_clear_ack(cdb2_hndl_tp *hndl); diff --git a/tests/test_dbapi2.py b/tests/test_dbapi2.py index 40e33fa..25409c4 100644 --- a/tests/test_dbapi2.py +++ b/tests/test_dbapi2.py @@ -36,6 +36,7 @@ import pytest import datetime import pytz +import re from functools import partial from unittest.mock import patch @@ -851,3 +852,66 @@ def test_interface_error_reading_result_set_after_commits(): ]) def test_finding_operation(statement, operation): assert _sql_operation(statement) == operation + + +@pytest.mark.parametrize( + "values", + [ + [5, 10], + ("hello", "hi"), + [0.25, 0.35, 0.25], + (b"123", b"456"), + ], +) +def test_parameter_binding_arrays(values): + # GIVEN + conn = connect("mattdb", "dev") + cursor = conn.cursor() + + # WHEN + cursor.execute("select * from carray(%(values)s)", dict(values=values)) + results = cursor.fetchall() + + # THEN + assert results == [[v] for v in values] + conn.close() + + +@pytest.mark.parametrize( + "values,exc_msg", + [ + ( + [], + "Can't bind list value [] for parameter 'values': " + + "ValueError: empty lists cannot be bound", + ), + ( + (), + "Can't bind tuple value () for parameter 'values': " + + "ValueError: empty tuples cannot be bound", + ), + ( + [1, "hello"], + "Can't bind list value [1, 'hello'] for parameter 'values': " + + "ValueError: all list elements must be the same type", + ), + ( + (1j,), + "Can't bind tuple value (1j,) for parameter 'values': " + + "ValueError: Cannot bind a tuple of complex", + ), + ( + ((1, 2, 3),), + "Can't bind tuple value ((1, 2, 3),) for parameter 'values': " + + "ValueError: Cannot bind a tuple of tuple", + ), + ], +) +def test_parameter_binding_invalid_arrays(values, exc_msg): + # GIVEN + conn = connect("mattdb", "dev") + cursor = conn.cursor() + + # WHEN/THEN + with pytest.raises(DataError, match=re.escape(exc_msg)): + cursor.execute("select * from carray(%(values)s)", dict(values=values)) From bd01ee012be5eaf25816388610a3afd9bb42bf09 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Tue, 28 Nov 2023 18:48:07 -0500 Subject: [PATCH 08/62] Fix a potential use-after-free bug It'd be hard to trigger, and would require a race between threads, but this defends against the issue. Signed-off-by: Matt Wozniski --- comdb2/_ccdb2.pyx | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/comdb2/_ccdb2.pyx b/comdb2/_ccdb2.pyx index bcaca37..2b508fe 100644 --- a/comdb2/_ccdb2.pyx +++ b/comdb2/_ccdb2.pyx @@ -181,9 +181,17 @@ cdef class _ParameterValue(object): (self.data)[l_index] = obj[l_index] return elif all(isinstance(ele, bytes) for ele in obj): + if isinstance(obj, tuple): + owner = obj + else: + # Copy the list into a tuple. If we referenced the list + # directly, another thread could remove an element when + # we drop the GIL, invalidating a pointer we're holding + owner = tuple(obj) + self.type = lib.CDB2_BLOB self.size = sizeof(blob_descriptor) - self.owner = obj + self.owner = owner self.data = PyMem_Malloc(self.list_size * self.size) for l_index in range(self.list_size): (self.data)[l_index].size = len(obj[l_index]) From a0b6893d699334d1d69df85bde0393f4d7fa87d4 Mon Sep 17 00:00:00 2001 From: Gus Monod Date: Tue, 28 Nov 2023 09:07:10 -0500 Subject: [PATCH 09/62] Prepare for release of `1.6.1` Signed-off-by: Gus Monod --- comdb2/_about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comdb2/_about.py b/comdb2/_about.py index 1180e5a..d719509 100644 --- a/comdb2/_about.py +++ b/comdb2/_about.py @@ -1,2 +1,2 @@ __all__ = ['__version__'] -__version__ = "1.6.0" +__version__ = "1.6.1" From 456576e929df1d363e0605ffe2201efc6ee3bb16 Mon Sep 17 00:00:00 2001 From: Gus Monod Date: Fri, 1 Dec 2023 16:17:15 -0500 Subject: [PATCH 10/62] Fix typo Fix typo Signed-off-by: Gus Monod --- docs/tips.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tips.rst b/docs/tips.rst index 5dc9afd..f4e91ff 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -11,7 +11,7 @@ Best Practices, Tips, and Tricks a unicode string should have been passed. An error message saying "incompatible values from SQL string of length 3 to - bbytearray field 'bar'" most likely means that you passed a unicode string + bytearray field 'bar'" most likely means that you passed a unicode string where a byte string should have been passed. Note the byte string must be *exactly* the length of the column. No padding is performed automatically, unless the column is declared with a ``dbpad`` attribute in the csc2 schema From 448e0c9f8352f165c9aa33be9caacf44e7f03775 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Tue, 23 Jan 2024 11:31:07 -0500 Subject: [PATCH 11/62] Update type annotations to allow binding arrays This update was missed when support for binding arrays was introduced. Signed-off-by: Matt Wozniski --- comdb2/cdb2.pyi | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/comdb2/cdb2.pyi b/comdb2/cdb2.pyi index c228b01..9a8de6f 100644 --- a/comdb2/cdb2.pyi +++ b/comdb2/cdb2.pyi @@ -23,6 +23,7 @@ from typing import Mapping from typing import NamedTuple from typing import Optional from typing import Text +from typing import Tuple from typing import Union from typing import overload @@ -106,7 +107,23 @@ ERROR_CODE: Dict[Text, int] TYPE: Dict[Text, int] HANDLE_FLAGS: Dict[Text, int] -Value = Union[None, int, float, bytes, Text, datetime, DatetimeUs] +Value = Union[ + None, + int, + float, + bytes, + Text, + datetime, + DatetimeUs, + List[int], + List[float], + List[bytes], + List[str], + Tuple[int, ...], + Tuple[float, ...], + Tuple[bytes, ...], + Tuple[str, ...], +] Row = Any From 060aea92f12b02abd1f325d6d6b4b744de11aa3c Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Fri, 1 Mar 2024 10:47:12 -0500 Subject: [PATCH 12/62] tests: Fix a test that fails on leap days This test takes the current date and time and switches the year to a hardcoded one. On leap day, that fails because the hardcoded year didn't have a February 29th. Update the test to use a hardcoded year that did. Signed-off-by: Matt Wozniski --- tests/test_cdb2_datetimeus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cdb2_datetimeus.py b/tests/test_cdb2_datetimeus.py index 36601d6..7266ba0 100644 --- a/tests/test_cdb2_datetimeus.py +++ b/tests/test_cdb2_datetimeus.py @@ -114,7 +114,7 @@ def check(obj): check(cdb2.DatetimeUs.now() - datetime.timedelta(0)) check(cdb2.DatetimeUs.now() + datetime.timedelta(0)) check(datetime.timedelta(0) + cdb2.DatetimeUs.now()) - check(cdb2.DatetimeUs.now().replace(year=2015)) + check(cdb2.DatetimeUs.now().replace(year=2016)) check(cdb2.DatetimeUs.now().replace(tzinfo=new_york)) check(cdb2.DatetimeUs.now(utc).astimezone(new_york)) From 7822ace22f2d279187b6daff4e147d7de2f47644 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Tue, 26 Mar 2024 10:07:45 -0400 Subject: [PATCH 13/62] ci: Explicitly remove Comdb2 build artifacts actions/checkout@v4 is getting a "permission denied" error when attempting to remove some of these files. Remove them ourselves as a super-user. Signed-off-by: Matt Wozniski --- .github/workflows/build.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 38a819b..0430d55 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -87,7 +87,8 @@ jobs: libunwind-dev zlib1g-dev && tar xvf bloomberg-comdb2.tar.gz && - (cd bloomberg-comdb2/build && sudo make install) + (cd bloomberg-comdb2/build && sudo make install) && + sudo rm -rf bloomberg-comdb2/ bloomberg-comdb2.tar.gz ' - uses: actions/checkout@v4 - name: 'Set up Python 3.7' @@ -136,7 +137,8 @@ jobs: libunwind-dev zlib1g-dev && tar xvf bloomberg-comdb2.tar.gz && - (cd bloomberg-comdb2/build && sudo make install) + (cd bloomberg-comdb2/build && sudo make install) && + sudo rm -rf bloomberg-comdb2/ bloomberg-comdb2.tar.gz ' - name: Start local comdb2 instance run: ' From 4df6e82b8481a47af9973516c794a5b5471afadc Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Mon, 25 Mar 2024 17:37:38 -0400 Subject: [PATCH 14/62] ci: Build docs, and publish them for releases Signed-off-by: Matt Wozniski --- .github/workflows/docs.yml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/docs.yml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..05b3bec --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,37 @@ +name: Docs + +on: + push: + pull_request: + release: + types: + - published + schedule: + # At 12:00 on every day-of-month + - cron: "0 12 */1 * *" + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + docs: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install Python dependencies + run: python3 -m pip install sphinx sphinx_rtd_theme + - name: Build docs + run: python -m sphinx docs html -a -E -n -v -W --keep-going + - name: Publish docs to GitHub Pages + if: github.event_name == 'release' && github.event.action == 'published' + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: html + single-commit: true From b3ad672404152907f87115871418cbe8953c245b Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Tue, 26 Mar 2024 18:02:21 -0400 Subject: [PATCH 15/62] Avoid a `*` import This is the only thing `ruff check` complains about. Signed-off-by: Matt Wozniski --- comdb2/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/comdb2/__init__.py b/comdb2/__init__.py index 2de7db9..c6382cc 100644 --- a/comdb2/__init__.py +++ b/comdb2/__init__.py @@ -28,4 +28,6 @@ connections, this module may be simpler to get started with. """ -from ._about import * +from ._about import __version__ + +__all__ = ["__version__"] From cff4fa004f61bec4e878fffcd3f69277163a76a4 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Tue, 26 Mar 2024 18:55:22 -0400 Subject: [PATCH 16/62] Merge type stubs into our Python code Now that we no longer support Python 2.7, we no longer benefit from maintaining typing information in stub files, and can instead use type annotations. This was partially performed by `merge-pyi` and `pyupgrade --py37-plus`, but partially by hand, and needs to be checked carefully. Signed-off-by: Matt Wozniski --- comdb2/_cdb2_types.py | 35 ++++---- comdb2/cdb2.py | 43 ++++++--- comdb2/cdb2.pyi | 162 ---------------------------------- comdb2/dbapi2.py | 68 +++++++++------ comdb2/dbapi2.pyi | 198 ------------------------------------------ comdb2/factories.py | 9 +- comdb2/factories.pyi | 26 ------ 7 files changed, 101 insertions(+), 440 deletions(-) delete mode 100644 comdb2/cdb2.pyi delete mode 100644 comdb2/dbapi2.pyi delete mode 100644 comdb2/factories.pyi diff --git a/comdb2/_cdb2_types.py b/comdb2/_cdb2_types.py index 004febb..6ed264f 100644 --- a/comdb2/_cdb2_types.py +++ b/comdb2/_cdb2_types.py @@ -9,8 +9,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from collections import namedtuple -from datetime import datetime +from __future__ import annotations + +import datetime +from typing import NamedTuple __name__ = 'comdb2.cdb2' @@ -34,7 +36,7 @@ class Error(RuntimeError): error_message (str): The string returned by cdb2api's ``cdb2_errstr`` after the failed call. """ - def __init__(self, error_code, error_message): + def __init__(self, error_code: int, error_message: str) -> None: if not(isinstance(error_message, str)): error_message = _errstr(error_message) self.error_code = error_code @@ -42,8 +44,7 @@ def __init__(self, error_code, error_message): super().__init__(error_code, error_message) -class Effects(namedtuple('Effects', - "num_affected num_selected num_updated num_deleted num_inserted")): +class Effects(NamedTuple): """Type used to represent the count of rows affected by a SQL query. An object of this type is returned by `Handle.get_effects`. @@ -55,10 +56,14 @@ class Effects(namedtuple('Effects', num_deleted (int): The number of rows that were deleted. num_inserted (int): The number of rows that were inserted. """ - __slots__ = () + num_affected: int + num_selected: int + num_updated: int + num_deleted: int + num_inserted: int -class DatetimeUs(datetime): +class DatetimeUs(datetime.datetime): """Provides a distinct representation for Comdb2's DATETIMEUS type. Historically, Comdb2 provided a DATETIME type with millisecond precision. @@ -81,7 +86,7 @@ class DatetimeUs(datetime): constructors inherited from `datetime.datetime`. """ @classmethod - def fromdatetime(cls, dt): + def fromdatetime(cls, dt: datetime.datetime) -> DatetimeUs: """Return a `DatetimeUs` copied from a given `datetime.datetime`""" fold = getattr(dt, 'fold', None) kwargs = {} @@ -92,28 +97,28 @@ def fromdatetime(cls, dt): dt.hour, dt.minute, dt.second, dt.microsecond, dt.tzinfo, **kwargs) - def __add__(self, other): + def __add__(self, other: datetime.timedelta) -> DatetimeUs: ret = super().__add__(other) - if isinstance(ret, datetime): + if isinstance(ret, datetime.datetime): return DatetimeUs.fromdatetime(ret) return ret # must be a timedelta - def __sub__(self, other): + def __sub__(self, other: datetime.timedelta) -> DatetimeUs: ret = super().__sub__(other) - if isinstance(ret, datetime): + if isinstance(ret, datetime.datetime): return DatetimeUs.fromdatetime(ret) return ret # must be a timedelta - def __radd__(self, other): + def __radd__(self, other: datetime.timedelta) -> DatetimeUs: return self + other @classmethod - def now(cls, tz=None): + def now(cls, tz: datetime.tzinfo | None=None) -> DatetimeUs: ret = super().now(tz) return DatetimeUs.fromdatetime(ret) @classmethod - def fromtimestamp(cls, timestamp, tz=None): + def fromtimestamp(cls, timestamp: float, tz: datetime.tzinfo | None=None) -> DatetimeUs: ret = super().fromtimestamp(timestamp, tz) return DatetimeUs.fromdatetime(ret) diff --git a/comdb2/cdb2.py b/comdb2/cdb2.py index 4ecd51a..0d755ed 100644 --- a/comdb2/cdb2.py +++ b/comdb2/cdb2.py @@ -128,12 +128,35 @@ Make sure to carefully read :ref:`String and Blob Types` on that page. """ +from __future__ import annotations + +import datetime +from typing import Any, Callable, Iterator, List, Mapping, Tuple, Union from ._cdb2_types import Error, Effects, DatetimeUs from ._ccdb2 import Handle as CHandle __all__ = ['Error', 'Handle', 'Effects', 'DatetimeUs', 'ERROR_CODE', 'TYPE', 'HANDLE_FLAGS'] +Value = Union[ + None, + int, + float, + bytes, + str, + datetime.datetime, + DatetimeUs, + List[int], + List[float], + List[bytes], + List[str], + Tuple[int, ...], + Tuple[float, ...], + Tuple[bytes, ...], + Tuple[str, ...], +] +Row = Any + # Pull all comdb2 error codes from cdb2api.h into our namespace ERROR_CODE = {'CONNECT_ERROR' : -1, 'NOTCONNECTED' : -2, @@ -247,8 +270,8 @@ class Handle: use a machine-specific default. """ - def __init__(self, database_name, tier="default", flags=0, tz='UTC', - host=None): + def __init__(self, database_name: str | bytes, tier: str | bytes="default", flags: int=0, tz: str='UTC', + host: str | bytes | None=None) -> None: if host is not None: if tier != "default": raise Error(ERROR_CODE['NOTSUPPORTED'], @@ -263,7 +286,7 @@ def __init__(self, database_name, tier="default", flags=0, tz='UTC', self._hndl.execute("set timezone %s" % tz, {}) self._cursor = iter([]) - def close(self, ack_current_event=True): + def close(self, ack_current_event: bool=True) -> None: """Gracefully close the Comdb2 connection. Once a `Handle` has been closed, no further operations may be performed @@ -294,7 +317,7 @@ def close(self, ack_current_event=True): self._hndl.close(ack_current_event=ack_current_event) @property - def row_factory(self): + def row_factory(self) -> Callable[[list[str]], Callable[[list[Value]], Row]]: """Factory used when constructing result rows. By default, or when set to ``None``, each row is returned as a `list` @@ -314,10 +337,10 @@ def row_factory(self): return self._hndl.row_factory @row_factory.setter - def row_factory(self, value): + def row_factory(self, value: Callable[[list[str]], Callable[[list[Value]], Row]]) -> None: self._hndl.row_factory = value - def execute(self, sql, parameters=None): + def execute(self, sql: str | bytes, parameters: Mapping[str, Value] | None=None) -> Handle: """Execute a database operation (query or command). The ``sql`` string may have placeholders for parameters to be passed. @@ -353,7 +376,7 @@ def execute(self, sql, parameters=None): self._cursor = iter(self._hndl.execute(sql, parameters)) return self - def __iter__(self): + def __iter__(self) -> Iterator[Row]: """Iterate over all remaining rows in the current result set. By default each row is returned as a `list`, where the elements in the @@ -376,7 +399,7 @@ def __next__(self): return next(self._cursor) next = __next__ - def get_effects(self): + def get_effects(self) -> Effects: """Return counts of rows affected by executed queries. Within a transaction, these counts are a running total from the start @@ -399,7 +422,7 @@ def get_effects(self): """ return self._hndl.get_effects() - def column_names(self): + def column_names(self) -> list[str]: """Returns the names of the columns of the current result set. Returns: @@ -407,7 +430,7 @@ def column_names(self): """ return self._hndl.column_names() - def column_types(self): + def column_types(self) -> list[int]: """Returns the type codes of the columns of the current result set. Returns: diff --git a/comdb2/cdb2.pyi b/comdb2/cdb2.pyi deleted file mode 100644 index 9a8de6f..0000000 --- a/comdb2/cdb2.pyi +++ /dev/null @@ -1,162 +0,0 @@ -# Copyright 2021 Bloomberg Finance L.P. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys - -from datetime import datetime -from datetime import timedelta -from datetime import tzinfo -from typing import Any -from typing import Callable -from typing import Dict -from typing import Iterator -from typing import List -from typing import Mapping -from typing import NamedTuple -from typing import Optional -from typing import Text -from typing import Tuple -from typing import Union -from typing import overload - - -class Error(RuntimeError): - error_code: int - error_message: str - - def __init__(self, error_code: int, error_message: str) -> None: ... - - -class Effects(NamedTuple): - num_affected: int - num_selected: int - num_updated: int - num_deleted: int - num_inserted: int - - -class DatetimeUs(datetime): - @classmethod - def fromdatetime(cls, dt: datetime) -> DatetimeUs: ... - - def __add__(self, other: timedelta) -> DatetimeUs: ... - - # XXX: datetime derives from date, which provides a __sub__ for a date rhs. - # mypy doesn't like that datetime and its subclasses don't provide it. - @overload # type: ignore[override] - def __sub__( - self, - other: datetime, - ) -> timedelta: ... - - @overload - def __sub__(self, other: timedelta) -> DatetimeUs: ... - - def __radd__(self, other: timedelta) -> DatetimeUs: ... - - @classmethod - def now(cls, tz: Optional[tzinfo] = ...) -> DatetimeUs: ... - - @classmethod - def fromtimestamp( - cls, timestamp: float, tz: Optional[tzinfo] = ... - ) -> DatetimeUs: ... - - if sys.version_info >= (3, 3): - def astimezone(self, tz: Optional[tzinfo]=...) -> DatetimeUs: ... - else: - def astimezone(self, tz: tzinfo) -> DatetimeUs: ... - - if sys.version_info >= (3, 6): - def replace( - self, - year: int = ..., - month: int = ..., - day: int = ..., - hour: int = ..., - minute: int = ..., - second: int = ..., - microsecond: int = ..., - tzinfo: Optional[tzinfo] = ..., - *, - fold: int = ..., - ) -> DatetimeUs: ... - else: - def replace( - self, - year: int = ..., - month: int = ..., - day: int = ..., - hour: int = ..., - minute: int = ..., - second: int = ..., - microsecond: int = ..., - tzinfo: Optional[tzinfo] = ..., - ) -> DatetimeUs: ... - - -ERROR_CODE: Dict[Text, int] -TYPE: Dict[Text, int] -HANDLE_FLAGS: Dict[Text, int] - -Value = Union[ - None, - int, - float, - bytes, - Text, - datetime, - DatetimeUs, - List[int], - List[float], - List[bytes], - List[str], - Tuple[int, ...], - Tuple[float, ...], - Tuple[bytes, ...], - Tuple[str, ...], -] -Row = Any - - -class Handle: - def __init__( - self, - database_name: Union[Text, bytes], - tier: Union[Text, bytes] = ..., - flags: int = ..., - tz: str = ..., - host: Optional[Union[Text, bytes]] = ..., - ) -> None: ... - - def close(self, *, ack_current_event=...) -> None: ... - - @property - def row_factory(self) -> Callable[[List[Text]], Callable[[List[Value]], Row]]: ... - - @row_factory.setter - def row_factory( - self, value: Callable[[List[Text]], Callable[[List[Value]], Row]] - ) -> None: ... - - def execute( - self, - sql: Union[Text, bytes], - parameters: Optional[Mapping[Text, Value]] = ..., - ) -> Handle: ... - - def __iter__(self) -> Iterator[Row]: ... - - def get_effects(self) -> Effects: ... - - def column_names(self) -> List[Text]: ... - - def column_types(self) -> List[int]: ... diff --git a/comdb2/dbapi2.py b/comdb2/dbapi2.py index 3aba62e..06aedd7 100644 --- a/comdb2/dbapi2.py +++ b/comdb2/dbapi2.py @@ -240,6 +240,8 @@ cursor for that connection. """ +from __future__ import annotations + import functools import itertools import weakref @@ -247,6 +249,8 @@ import re from . import cdb2 +from .cdb2 import Row, Value +from typing import Any, Callable, Iterator, Mapping, Sequence __all__ = ['apilevel', 'threadsafety', 'paramstyle', 'connect', 'Connection', 'Cursor', @@ -317,7 +321,7 @@ def __repr__(self): return 'TypeObject' + str(self.value_names) -def _binary(string): +def _binary(string: str | bytes) -> bytes: if isinstance(string, str): return string.encode('utf-8') return bytes(string) @@ -544,7 +548,12 @@ def _modifies_rows(operation): return operation in ('commit', 'insert', 'update', 'delete') -def connect(*args, **kwargs): +def connect( + database_name: str | bytes, + tier: str | bytes="default", + autocommit: bool=False, + host: str | bytes | None=None +) -> Connection: """Establish a connection to a Comdb2 database. All arguments are passed directly through to the `Connection` constructor. @@ -558,7 +567,12 @@ def connect(*args, **kwargs): Returns: Connection: A handle for the newly established connection. """ - return Connection(*args, **kwargs) + return Connection( + database_name=database_name, + tier=tier, + autocommit=autocommit, + host=host, + ) class Connection: @@ -619,8 +633,8 @@ class Connection: statements, disabling DB-API 2.0's automatic implicit transactions. """ - def __init__(self, database_name, tier="default", autocommit=False, - host=None): + def __init__(self, database_name: str | bytes, tier: str | bytes="default", autocommit: bool=False, + host: str | bytes | None=None) -> None: if host is not None and tier != "default": raise InterfaceError("Connecting to a host by name and to a " "cluster by tier are mutually exclusive") @@ -638,7 +652,7 @@ def _check_closed(self): raise InterfaceError("Attempted to use a closed Connection") @property - def row_factory(self): + def row_factory(self) -> Callable[[list[str]], Callable[[list[Value]], Row]]: """Factory used when constructing result rows. By default, or when set to ``None``, each row is returned as a `list` @@ -660,7 +674,7 @@ def row_factory(self): return self._hndl.row_factory @row_factory.setter - def row_factory(self, value): + def row_factory(self, value: Callable[[list[str]], Callable[[list[Value]], Row]]) -> None: self._check_closed() self._hndl.row_factory = value @@ -678,7 +692,7 @@ def _execute(self, operation): cursor = self.cursor() cursor._execute(operation, operation) - def close(self, ack_current_event=True): + def close(self, ack_current_event: bool=True) -> None: """Gracefully close the Comdb2 connection. Once a `Connection` has been closed, no further operations may be @@ -711,7 +725,7 @@ def close(self, ack_current_event=True): self._hndl.close(ack_current_event=ack_current_event) self._hndl = None - def commit(self): + def commit(self) -> None: """Commit any pending transaction to the database. This method will fail if the `Connection` is in ``autocommit`` mode and @@ -720,7 +734,7 @@ def commit(self): self._check_closed() self._execute("commit") - def rollback(self): + def rollback(self) -> None: """Rollback the current transaction. This method will fail if the `Connection` is in ``autocommit`` mode and @@ -736,7 +750,7 @@ def rollback(self): self._check_closed() self._execute("rollback") - def cursor(self): + def cursor(self) -> Cursor: """Return a new `Cursor` for this connection. This calls `Cursor.close` on any outstanding `Cursor`; only one @@ -792,7 +806,7 @@ class Cursor: 'rollback': "Use Connection.rollback to roll back transactions", } - def __init__(self, conn): + def __init__(self, conn: Connection) -> None: self._arraysize = 1 self._conn = conn self._hndl = conn._hndl @@ -805,7 +819,7 @@ def _check_closed(self): raise InterfaceError("Attempted to use a closed cursor") @property - def arraysize(self): + def arraysize(self) -> int: """Controls the number of rows to fetch at a time with `fetchmany`. The default is ``1``, meaning that a single row will be fetched at @@ -814,11 +828,11 @@ def arraysize(self): return self._arraysize @arraysize.setter - def arraysize(self, value): + def arraysize(self, value: int) -> None: self._arraysize = value @property - def description(self): + def description(self) -> tuple[tuple[str, object, None, None, None, None, None], ...]: """Provides the name and type of each column in the latest result set. This read-only attribute will contain one element per column in the @@ -853,7 +867,7 @@ def description(self): return self._description @property - def rowcount(self): + def rowcount(self) -> int: """Provides the count of rows modified by the last transaction. For `Cursor` objects on a `Connection` that is not using ``autocommit`` @@ -882,12 +896,12 @@ def rowcount(self): # Optional DB API Extension @property - def connection(self): + def connection(self) -> Connection: """Return a reference to the `Connection` that this `Cursor` uses.""" self._check_closed() return self._conn - def close(self): + def close(self) -> None: """Close the cursor now. From this point forward an exception will be raised if any @@ -903,7 +917,7 @@ def close(self): self._description = None self._closed = True - def callproc(self, procname, parameters): + def callproc(self, procname: str, parameters: Sequence[Value]) -> Sequence[Value]: """Call a stored procedure with the given name. The ``parameters`` sequence must contain one entry for each argument @@ -932,7 +946,7 @@ def callproc(self, procname, parameters): self.execute(sql, params_as_dict) return parameters[:] - def execute(self, sql, parameters=None): + def execute(self, sql: str, parameters: Mapping[str, Value] | None=None) -> Cursor: """Execute a database operation (query or command). The ``sql`` string must be provided as a Python format string, with @@ -984,7 +998,7 @@ def execute(self, sql, parameters=None): # return an iterable over the rows, but this isn't portable across DBs. return self - def executemany(self, sql, seq_of_parameters): + def executemany(self, sql: str, seq_of_parameters: Sequence[Mapping[str, Value]]) -> None: """Execute the same SQL statement repeatedly with different parameters. This is currently equivalent to calling execute multiple times, once @@ -1045,11 +1059,11 @@ def _execute(self, operation, sql, parameters=None): # an explicit transaction. We can get the count of affected rows. self._update_rowcount() - def setinputsizes(self, sizes): + def setinputsizes(self, sizes: Sequence[Any]) -> None: """No-op; implemented for PEP-249 compliance.""" self._check_closed() - def setoutputsize(self, size, column=None): + def setoutputsize(self, size: Any, column: int=None) -> None: """No-op; implemented for PEP-249 compliance.""" self._check_closed() @@ -1067,7 +1081,7 @@ def _load_description(self): if not self._description: self._description = None - def fetchone(self): + def fetchone(self) -> Row | None: """Fetch the next row of the current result set. Returns: @@ -1082,7 +1096,7 @@ def fetchone(self): except StopIteration: return None - def fetchmany(self, n=None): + def fetchmany(self, n: int | None=None) -> Sequence[Row]: """Fetch the next set of rows of the current result set. Args: @@ -1102,7 +1116,7 @@ def fetchmany(self, n=None): n = self._arraysize return [x for x in itertools.islice(self, 0, n)] - def fetchall(self): + def fetchall(self) -> Sequence[Row]: """Fetch all remaining rows of the current result set. Returns: @@ -1115,7 +1129,7 @@ def fetchall(self): return [x for x in self] # Optional DB API Extension - def __iter__(self): + def __iter__(self) -> Iterator[Row]: """Iterate over all rows in a result set. By default each row is returned as a `list`, where the elements in the diff --git a/comdb2/dbapi2.pyi b/comdb2/dbapi2.pyi deleted file mode 100644 index f8db95f..0000000 --- a/comdb2/dbapi2.pyi +++ /dev/null @@ -1,198 +0,0 @@ -# Copyright 2021 Bloomberg Finance L.P. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import datetime - -from typing import Any -from typing import Callable -from typing import Iterator -from typing import List -from typing import Mapping -from typing import Optional -from typing import Sequence -from typing import Text -from typing import Tuple -from typing import Union - -from . import cdb2 - -apilevel: str -threadsafety: int -paramstyle: str - -STRING: object -BINARY: object -NUMBER: object -DATETIME: object -ROWID: object - - -def Binary(string: Union[Text, bytes]) -> bytes: ... - - -Datetime = datetime.datetime -DatetimeUs = cdb2.DatetimeUs -Timestamp = Datetime -TimestampUs = DatetimeUs - -DatetimeFromTicks = Datetime.fromtimestamp -DatetimeUsFromTicks = DatetimeUs.fromtimestamp -TimestampFromTicks = Timestamp.fromtimestamp -TimestampUsFromTicks = TimestampUs.fromtimestamp - -Value = cdb2.Value -Row = cdb2.Row - - -class Error(Exception): - ... - - -class Warning(Exception): - ... - - -class InterfaceError(Error): - ... - - -class DatabaseError(Error): - ... - - -class InternalError(DatabaseError): - ... - - -class OperationalError(DatabaseError): - ... - - -class ProgrammingError(DatabaseError): - ... - - -class IntegrityError(DatabaseError): - ... - - -class UniqueKeyConstraintError(IntegrityError): - ... - - -class ForeignKeyConstraintError(IntegrityError): - ... - - -class NonNullConstraintError(IntegrityError): - ... - - -class DataError(DatabaseError): - ... - - -class NotSupportedError(DatabaseError): - ... - - -def connect( - database_name: Union[Text, bytes], - tier: Union[Text, bytes] = ..., - autocommit: bool = ..., - host: Optional[Union[Text, bytes]] = ..., -) -> Connection: ... - - -class Connection: - def __init__( - self, - database_name: Union[Text, bytes], - tier: Union[Text, bytes] = ..., - autocommit: bool = ..., - host: Optional[Union[Text, bytes]] = ..., - ) -> None: ... - - @property - def row_factory(self) -> Callable[[List[Text]], Callable[[List[Value]], Row]]: ... - - @row_factory.setter - def row_factory( - self, value: Callable[[List[Text]], Callable[[List[Value]], Row]] - ) -> None: ... - - def close(self, *, ack_current_event=...) -> None: ... - - def commit(self) -> None: ... - - def rollback(self) -> None: ... - - def cursor(self) -> Cursor: ... - - Error = Error - Warning = Warning - InterfaceError = InterfaceError - DatabaseError = DatabaseError - InternalError = InternalError - OperationalError = OperationalError - ProgrammingError = ProgrammingError - IntegrityError = IntegrityError - DataError = DataError - NotSupportedError = NotSupportedError - - -class Cursor: - @property - def arraysize(self) -> int: ... - - @arraysize.setter - def arraysize(self, value: int) -> None: ... - - @property - def description(self) -> Tuple[Text, object, None, None, None, None, None]: ... - - @property - def rowcount(self) -> int: ... - - @property - def connection(self) -> Connection: ... - - def close(self) -> None: ... - - def callproc( - self, - procname: Union[str, Text], - parameters: Sequence[Value], - ) -> Sequence[Value]: ... - - def execute( - self, - sql: Union[str, Text], - parameters: Optional[Mapping[Text, Value]] = None, - ) -> Cursor: ... - - def executemany( - self, - sql: str, - seq_of_parameters: Sequence[Mapping[Text, Value]], - ) -> None: ... - - def setinputsizes(self, sizes: Sequence[Any]) -> None: ... - - def setoutputsize(self, size: Any, column: int = ...) -> None: ... - - def fetchone(self) -> Optional[Row]: ... - - def fetchmany(self, n: Optional[int] = ...) -> Sequence[Row]: ... - - def fetchall(self) -> Sequence[Row]: ... - - def __iter__(self) -> Iterator[Row]: ... diff --git a/comdb2/factories.py b/comdb2/factories.py index d518aae..fb8ace2 100644 --- a/comdb2/factories.py +++ b/comdb2/factories.py @@ -21,11 +21,16 @@ A factory function will be called with a list of column names, and must return a callable that will be called once per row with a list of column values. """ + +from __future__ import annotations + from collections import namedtuple from collections import Counter +from .cdb2 import Value +from typing import Callable, NamedTuple -def namedtuple_row_factory(col_names): +def namedtuple_row_factory(col_names: list[str]) -> Callable[[list[Value]], NamedTuple]: """Return each result row as a `collections.namedtuple`. The fields of the `~collections.namedtuple` are set to the names of the @@ -85,7 +90,7 @@ def namedtuple_row_factory(col_names): raise -def dict_row_factory(col_names): +def dict_row_factory(col_names: list[str]) -> Callable[[list[Value]], dict[str, Value]]: """Return each result row as a `dict` mapping column names to values. Note: diff --git a/comdb2/factories.pyi b/comdb2/factories.pyi deleted file mode 100644 index 7ca9edf..0000000 --- a/comdb2/factories.pyi +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2021 Bloomberg Finance L.P. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Callable -from typing import Dict -from typing import NamedTuple -from typing import List -from typing import Text - -from .cdb2 import Value - -def namedtuple_row_factory( - col_names: List[Text], -) -> Callable[[List[Value]], NamedTuple]: ... - -def dict_row_factory( - col_names: List[Text] -) -> Callable[[List[Value]], Dict[Text, Value]]: ... From ec7b34573c91db5ea0bc2a0e50f9e01f57e2960c Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Tue, 26 Mar 2024 19:08:27 -0400 Subject: [PATCH 17/62] Format with `ruff` This commit was entirely automatic, from `ruff format`. Signed-off-by: Matt Wozniski --- comdb2/_about.py | 2 +- comdb2/_cdb2_types.py | 47 ++-- comdb2/cdb2.py | 147 +++++++------ comdb2/dbapi2.py | 223 +++++++++++-------- comdb2/factories.py | 6 +- docs/conf.py | 119 +++++----- setup.py | 28 +-- tests/test_cdb2.py | 123 +++++------ tests/test_cdb2_datetimeus.py | 15 +- tests/test_dbapi2.py | 394 +++++++++++++++++++--------------- 10 files changed, 623 insertions(+), 481 deletions(-) diff --git a/comdb2/_about.py b/comdb2/_about.py index d719509..73f175b 100644 --- a/comdb2/_about.py +++ b/comdb2/_about.py @@ -1,2 +1,2 @@ -__all__ = ['__version__'] +__all__ = ["__version__"] __version__ = "1.6.1" diff --git a/comdb2/_cdb2_types.py b/comdb2/_cdb2_types.py index 6ed264f..394a93b 100644 --- a/comdb2/_cdb2_types.py +++ b/comdb2/_cdb2_types.py @@ -14,18 +14,18 @@ import datetime from typing import NamedTuple -__name__ = 'comdb2.cdb2' +__name__ = "comdb2.cdb2" def _errstr(msg): try: - return msg.decode('utf-8') + return msg.decode("utf-8") except UnicodeDecodeError: # The DB's error strings aren't necessarily UTF-8. # If one isn't, it's preferable to mangle the error string than to # raise a UnicodeDecodeError (which would obscure the root cause). # Return a unicode string with \x escapes in place of non-ascii bytes. - return msg.decode('latin1').encode('unicode_escape').decode('ascii') + return msg.decode("latin1").encode("unicode_escape").decode("ascii") class Error(RuntimeError): @@ -36,8 +36,9 @@ class Error(RuntimeError): error_message (str): The string returned by cdb2api's ``cdb2_errstr`` after the failed call. """ + def __init__(self, error_code: int, error_message: str) -> None: - if not(isinstance(error_message, str)): + if not (isinstance(error_message, str)): error_message = _errstr(error_message) self.error_code = error_code self.error_message = error_message @@ -56,6 +57,7 @@ class Effects(NamedTuple): num_deleted (int): The number of rows that were deleted. num_inserted (int): The number of rows that were inserted. """ + num_affected: int num_selected: int num_updated: int @@ -85,40 +87,51 @@ class DatetimeUs(datetime.datetime): `.fromdatetime` alternate constructor, or any of the other alternate constructors inherited from `datetime.datetime`. """ + @classmethod - def fromdatetime(cls, dt: datetime.datetime) -> DatetimeUs: + def fromdatetime(cls, dt: datetime.datetime) -> DatetimeUs: """Return a `DatetimeUs` copied from a given `datetime.datetime`""" - fold = getattr(dt, 'fold', None) + fold = getattr(dt, "fold", None) kwargs = {} if fold is not None: - kwargs['fold'] = fold - - return DatetimeUs(dt.year, dt.month, dt.day, - dt.hour, dt.minute, dt.second, dt.microsecond, - dt.tzinfo, **kwargs) - - def __add__(self, other: datetime.timedelta) -> DatetimeUs: + kwargs["fold"] = fold + + return DatetimeUs( + dt.year, + dt.month, + dt.day, + dt.hour, + dt.minute, + dt.second, + dt.microsecond, + dt.tzinfo, + **kwargs, + ) + + def __add__(self, other: datetime.timedelta) -> DatetimeUs: ret = super().__add__(other) if isinstance(ret, datetime.datetime): return DatetimeUs.fromdatetime(ret) return ret # must be a timedelta - def __sub__(self, other: datetime.timedelta) -> DatetimeUs: + def __sub__(self, other: datetime.timedelta) -> DatetimeUs: ret = super().__sub__(other) if isinstance(ret, datetime.datetime): return DatetimeUs.fromdatetime(ret) return ret # must be a timedelta - def __radd__(self, other: datetime.timedelta) -> DatetimeUs: + def __radd__(self, other: datetime.timedelta) -> DatetimeUs: return self + other @classmethod - def now(cls, tz: datetime.tzinfo | None=None) -> DatetimeUs: + def now(cls, tz: datetime.tzinfo | None = None) -> DatetimeUs: ret = super().now(tz) return DatetimeUs.fromdatetime(ret) @classmethod - def fromtimestamp(cls, timestamp: float, tz: datetime.tzinfo | None=None) -> DatetimeUs: + def fromtimestamp( + cls, timestamp: float, tz: datetime.tzinfo | None = None + ) -> DatetimeUs: ret = super().fromtimestamp(timestamp, tz) return DatetimeUs.fromdatetime(ret) diff --git a/comdb2/cdb2.py b/comdb2/cdb2.py index 0d755ed..0666afb 100644 --- a/comdb2/cdb2.py +++ b/comdb2/cdb2.py @@ -135,8 +135,15 @@ from ._cdb2_types import Error, Effects, DatetimeUs from ._ccdb2 import Handle as CHandle -__all__ = ['Error', 'Handle', 'Effects', 'DatetimeUs', - 'ERROR_CODE', 'TYPE', 'HANDLE_FLAGS'] +__all__ = [ + "Error", + "Handle", + "Effects", + "DatetimeUs", + "ERROR_CODE", + "TYPE", + "HANDLE_FLAGS", +] Value = Union[ None, @@ -158,41 +165,42 @@ Row = Any # Pull all comdb2 error codes from cdb2api.h into our namespace -ERROR_CODE = {'CONNECT_ERROR' : -1, - 'NOTCONNECTED' : -2, - 'PREPARE_ERROR' : -3, - 'IO_ERROR' : -4, - 'INTERNAL' : -5, - 'NOSTATEMENT' : -6, - 'BADCOLUMN' : -7, - 'BADSTATE' : -8, - 'ASYNCERR' : -9, - 'INVALID_ID' : -12, - 'RECORD_OUT_OF_RANGE' : -13, - 'REJECTED' : -15, - 'STOPPED' : -16, - 'BADREQ' : -17, - 'DBCREATE_FAILED' : -18, - 'THREADPOOL_INTERNAL' : -20, - 'READONLY' : -21, - 'NOMASTER' : -101, - 'UNTAGGED_DATABASE' : -102, - 'CONSTRAINTS' : -103, - 'DEADLOCK' : 203, - 'TRAN_IO_ERROR' : -105, - 'ACCESS' : -106, - 'TRAN_MODE_UNSUPPORTED' : -107, - 'VERIFY_ERROR' : 2, - 'FKEY_VIOLATION' : 3, - 'NULL_CONSTRAINT' : 4, - 'CONV_FAIL' : 113, - 'NONKLESS' : 114, - 'MALLOC' : 115, - 'NOTSUPPORTED' : 116, - 'DUPLICATE' : 299, - 'TZNAME_FAIL' : 401, - 'UNKNOWN' : 300, - } +ERROR_CODE = { + "CONNECT_ERROR": -1, + "NOTCONNECTED": -2, + "PREPARE_ERROR": -3, + "IO_ERROR": -4, + "INTERNAL": -5, + "NOSTATEMENT": -6, + "BADCOLUMN": -7, + "BADSTATE": -8, + "ASYNCERR": -9, + "INVALID_ID": -12, + "RECORD_OUT_OF_RANGE": -13, + "REJECTED": -15, + "STOPPED": -16, + "BADREQ": -17, + "DBCREATE_FAILED": -18, + "THREADPOOL_INTERNAL": -20, + "READONLY": -21, + "NOMASTER": -101, + "UNTAGGED_DATABASE": -102, + "CONSTRAINTS": -103, + "DEADLOCK": 203, + "TRAN_IO_ERROR": -105, + "ACCESS": -106, + "TRAN_MODE_UNSUPPORTED": -107, + "VERIFY_ERROR": 2, + "FKEY_VIOLATION": 3, + "NULL_CONSTRAINT": 4, + "CONV_FAIL": 113, + "NONKLESS": 114, + "MALLOC": 115, + "NOTSUPPORTED": 116, + "DUPLICATE": 299, + "TZNAME_FAIL": 401, + "UNKNOWN": 300, +} """This dict maps all known Comdb2 error names to their respective values. The value returned in `Error.error_code` will generally be the value @@ -202,16 +210,17 @@ """ # Pull comdb2 column types from cdb2api.h into our namespace -TYPE = {'INTEGER' : 1, - 'REAL' : 2, - 'CSTRING' : 3, - 'BLOB' : 4, - 'DATETIME' : 6, - 'INTERVALYM' : 7, - 'INTERVALDS' : 8, - 'DATETIMEUS' : 9, - 'INTERVALDSUS' : 10, - } +TYPE = { + "INTEGER": 1, + "REAL": 2, + "CSTRING": 3, + "BLOB": 4, + "DATETIME": 6, + "INTERVALYM": 7, + "INTERVALDS": 8, + "DATETIMEUS": 9, + "INTERVALDSUS": 10, +} """This dict maps all known Comdb2 types to their enumeration value. Each value in the list returned by `Handle.column_types` will generally be the @@ -220,12 +229,13 @@ """ # Pull comdb2 handle flags from cdb2api.h into our namespace -HANDLE_FLAGS = {'READ_INTRANS_RESULTS' : 2, - 'DIRECT_CPU' : 4, - 'RANDOM' : 8, - 'RANDOMROOM' : 16, - 'ROOM' : 32, - } +HANDLE_FLAGS = { + "READ_INTRANS_RESULTS": 2, + "DIRECT_CPU": 4, + "RANDOM": 8, + "RANDOMROOM": 16, + "ROOM": 32, +} """This dict maps all known Comdb2 flags to their enumeration value. These values can be passed directly to `Handle`, though values not in this dict @@ -270,23 +280,31 @@ class Handle: use a machine-specific default. """ - def __init__(self, database_name: str | bytes, tier: str | bytes="default", flags: int=0, tz: str='UTC', - host: str | bytes | None=None) -> None: + def __init__( + self, + database_name: str | bytes, + tier: str | bytes = "default", + flags: int = 0, + tz: str = "UTC", + host: str | bytes | None = None, + ) -> None: if host is not None: if tier != "default": - raise Error(ERROR_CODE['NOTSUPPORTED'], - "Connecting to a host by name and to a " - "cluster by tier are mutually exclusive") + raise Error( + ERROR_CODE["NOTSUPPORTED"], + "Connecting to a host by name and to a " + "cluster by tier are mutually exclusive", + ) else: tier = host - flags |= HANDLE_FLAGS['DIRECT_CPU'] + flags |= HANDLE_FLAGS["DIRECT_CPU"] self._hndl = CHandle(database_name, tier, flags) if tz is not None: self._hndl.execute("set timezone %s" % tz, {}) self._cursor = iter([]) - def close(self, ack_current_event: bool=True) -> None: + def close(self, ack_current_event: bool = True) -> None: """Gracefully close the Comdb2 connection. Once a `Handle` has been closed, no further operations may be performed @@ -337,10 +355,14 @@ def row_factory(self) -> Callable[[list[str]], Callable[[list[Value]], Row]]: return self._hndl.row_factory @row_factory.setter - def row_factory(self, value: Callable[[list[str]], Callable[[list[Value]], Row]]) -> None: + def row_factory( + self, value: Callable[[list[str]], Callable[[list[Value]], Row]] + ) -> None: self._hndl.row_factory = value - def execute(self, sql: str | bytes, parameters: Mapping[str, Value] | None=None) -> Handle: + def execute( + self, sql: str | bytes, parameters: Mapping[str, Value] | None = None + ) -> Handle: """Execute a database operation (query or command). The ``sql`` string may have placeholders for parameters to be passed. @@ -397,6 +419,7 @@ def __next__(self): # compatibility for a user who did something like: # next(handle.execute("select 1")) return next(self._cursor) + next = __next__ def get_effects(self) -> Effects: diff --git a/comdb2/dbapi2.py b/comdb2/dbapi2.py index 06aedd7..6741a24 100644 --- a/comdb2/dbapi2.py +++ b/comdb2/dbapi2.py @@ -252,16 +252,40 @@ from .cdb2 import Row, Value from typing import Any, Callable, Iterator, Mapping, Sequence -__all__ = ['apilevel', 'threadsafety', 'paramstyle', - 'connect', 'Connection', 'Cursor', - 'STRING', 'BINARY', 'NUMBER', 'DATETIME', 'ROWID', - 'Datetime', 'DatetimeUs', 'Binary', 'Timestamp', 'TimestampUs', - 'DatetimeFromTicks', 'DatetimeUsFromTicks', 'TimestampFromTicks', - 'Error', 'Warning', 'InterfaceError', 'DatabaseError', - 'InternalError', 'OperationalError', 'ProgrammingError', - 'IntegrityError', 'DataError', 'NotSupportedError', - 'UniqueKeyConstraintError', 'ForeignKeyConstraintError', - 'NonNullConstraintError'] +__all__ = [ + "apilevel", + "threadsafety", + "paramstyle", + "connect", + "Connection", + "Cursor", + "STRING", + "BINARY", + "NUMBER", + "DATETIME", + "ROWID", + "Datetime", + "DatetimeUs", + "Binary", + "Timestamp", + "TimestampUs", + "DatetimeFromTicks", + "DatetimeUsFromTicks", + "TimestampFromTicks", + "Error", + "Warning", + "InterfaceError", + "DatabaseError", + "InternalError", + "OperationalError", + "ProgrammingError", + "IntegrityError", + "DataError", + "NotSupportedError", + "UniqueKeyConstraintError", + "ForeignKeyConstraintError", + "NonNullConstraintError", +] apilevel = "2.0" """This module conforms to the Python Database API Specification 2.0.""" @@ -302,7 +326,7 @@ """, re.VERBOSE | re.DOTALL | re.ASCII, ) -_VALID_SP_NAME = re.compile(r'^[A-Za-z0-9_.]+$') +_VALID_SP_NAME = re.compile(r"^[A-Za-z0-9_.]+$") @functools.total_ordering @@ -318,24 +342,25 @@ def __lt__(self, other): return self != other and other < self.values def __repr__(self): - return 'TypeObject' + str(self.value_names) + return "TypeObject" + str(self.value_names) def _binary(string: str | bytes) -> bytes: if isinstance(string, str): - return string.encode('utf-8') + return string.encode("utf-8") return bytes(string) -STRING = _TypeObject('CSTRING') + +STRING = _TypeObject("CSTRING") """The type codes of TEXT result columns compare equal to this constant.""" -BINARY = _TypeObject('BLOB') +BINARY = _TypeObject("BLOB") """The type codes of BLOB result columns compare equal to this constant.""" -NUMBER = _TypeObject('INTEGER', 'REAL') +NUMBER = _TypeObject("INTEGER", "REAL") """The type codes of numeric result columns compare equal to this constant.""" -DATETIME = _TypeObject('DATETIME', 'DATETIMEUS') +DATETIME = _TypeObject("DATETIME", "DATETIMEUS") """The type codes of datetime result columns compare equal to this constant.""" ROWID = STRING @@ -363,6 +388,7 @@ class Error(UserException): `Connection` objects, to simplify error handling in environments where multiple connections from different modules are used. """ + pass @@ -371,21 +397,25 @@ class Warning(UserException): This is required to exist by the DB-API interface, but we never raise it. """ + pass class InterfaceError(Error): """Exception raised for errors caused by misuse of this module.""" + pass class DatabaseError(Error): """Base class for all errors reported by the database.""" + pass class InternalError(DatabaseError): """Exception raised for internal errors reported by the database.""" + pass @@ -395,6 +425,7 @@ class OperationalError(DatabaseError): These errors are not necessarily the result of a bug either in the application or in the database - for example, dropped connections. """ + pass @@ -404,6 +435,7 @@ class ProgrammingError(DatabaseError): For example, this will be raised for syntactically incorrect SQL, or for passing a different number of parameters than are required by the query. """ + pass @@ -415,6 +447,7 @@ class IntegrityError(DatabaseError): an index may not have duplicates. Other types of constraint violations may raise this type directly. """ + pass @@ -427,6 +460,7 @@ class UniqueKeyConstraintError(IntegrityError): .. versionadded:: 1.1 """ + pass @@ -443,6 +477,7 @@ class ForeignKeyConstraintError(IntegrityError): .. versionadded:: 1.1 """ + pass @@ -455,6 +490,7 @@ class NonNullConstraintError(IntegrityError): .. versionadded:: 1.1 """ + pass @@ -465,65 +501,57 @@ class DataError(DatabaseError): of range for the column type that would store it, or if you specify an invalid timezone name for the connection. """ + pass class NotSupportedError(DatabaseError): """Exception raised when unsupported operations are attempted.""" + pass _EXCEPTION_BY_RC = { - cdb2.ERROR_CODE['CONNECT_ERROR'] : OperationalError, - cdb2.ERROR_CODE['NOTCONNECTED'] : ProgrammingError, - cdb2.ERROR_CODE['PREPARE_ERROR'] : ProgrammingError, - cdb2.ERROR_CODE['IO_ERROR'] : OperationalError, - cdb2.ERROR_CODE['INTERNAL'] : InternalError, - cdb2.ERROR_CODE['NOSTATEMENT'] : ProgrammingError, - cdb2.ERROR_CODE['BADCOLUMN'] : ProgrammingError, - cdb2.ERROR_CODE['BADSTATE'] : ProgrammingError, - cdb2.ERROR_CODE['ASYNCERR'] : OperationalError, - - cdb2.ERROR_CODE['INVALID_ID'] : InternalError, - cdb2.ERROR_CODE['RECORD_OUT_OF_RANGE'] : OperationalError, - - cdb2.ERROR_CODE['REJECTED'] : OperationalError, - cdb2.ERROR_CODE['STOPPED'] : OperationalError, - cdb2.ERROR_CODE['BADREQ'] : OperationalError, - cdb2.ERROR_CODE['DBCREATE_FAILED'] : OperationalError, - - cdb2.ERROR_CODE['THREADPOOL_INTERNAL'] : OperationalError, - cdb2.ERROR_CODE['READONLY'] : NotSupportedError, - - cdb2.ERROR_CODE['NOMASTER'] : InternalError, - cdb2.ERROR_CODE['UNTAGGED_DATABASE'] : NotSupportedError, - cdb2.ERROR_CODE['CONSTRAINTS'] : IntegrityError, - cdb2.ERROR_CODE['DEADLOCK'] : OperationalError, - - cdb2.ERROR_CODE['TRAN_IO_ERROR'] : OperationalError, - cdb2.ERROR_CODE['ACCESS'] : OperationalError, - - cdb2.ERROR_CODE['TRAN_MODE_UNSUPPORTED'] : NotSupportedError, - - cdb2.ERROR_CODE['VERIFY_ERROR'] : OperationalError, - cdb2.ERROR_CODE['FKEY_VIOLATION'] : ForeignKeyConstraintError, - cdb2.ERROR_CODE['NULL_CONSTRAINT'] : NonNullConstraintError, - - cdb2.ERROR_CODE['CONV_FAIL'] : DataError, - cdb2.ERROR_CODE['NONKLESS'] : NotSupportedError, - cdb2.ERROR_CODE['MALLOC'] : OperationalError, - cdb2.ERROR_CODE['NOTSUPPORTED'] : NotSupportedError, - - cdb2.ERROR_CODE['DUPLICATE'] : UniqueKeyConstraintError, - cdb2.ERROR_CODE['TZNAME_FAIL'] : DataError, - - cdb2.ERROR_CODE['UNKNOWN'] : OperationalError, + cdb2.ERROR_CODE["CONNECT_ERROR"]: OperationalError, + cdb2.ERROR_CODE["NOTCONNECTED"]: ProgrammingError, + cdb2.ERROR_CODE["PREPARE_ERROR"]: ProgrammingError, + cdb2.ERROR_CODE["IO_ERROR"]: OperationalError, + cdb2.ERROR_CODE["INTERNAL"]: InternalError, + cdb2.ERROR_CODE["NOSTATEMENT"]: ProgrammingError, + cdb2.ERROR_CODE["BADCOLUMN"]: ProgrammingError, + cdb2.ERROR_CODE["BADSTATE"]: ProgrammingError, + cdb2.ERROR_CODE["ASYNCERR"]: OperationalError, + cdb2.ERROR_CODE["INVALID_ID"]: InternalError, + cdb2.ERROR_CODE["RECORD_OUT_OF_RANGE"]: OperationalError, + cdb2.ERROR_CODE["REJECTED"]: OperationalError, + cdb2.ERROR_CODE["STOPPED"]: OperationalError, + cdb2.ERROR_CODE["BADREQ"]: OperationalError, + cdb2.ERROR_CODE["DBCREATE_FAILED"]: OperationalError, + cdb2.ERROR_CODE["THREADPOOL_INTERNAL"]: OperationalError, + cdb2.ERROR_CODE["READONLY"]: NotSupportedError, + cdb2.ERROR_CODE["NOMASTER"]: InternalError, + cdb2.ERROR_CODE["UNTAGGED_DATABASE"]: NotSupportedError, + cdb2.ERROR_CODE["CONSTRAINTS"]: IntegrityError, + cdb2.ERROR_CODE["DEADLOCK"]: OperationalError, + cdb2.ERROR_CODE["TRAN_IO_ERROR"]: OperationalError, + cdb2.ERROR_CODE["ACCESS"]: OperationalError, + cdb2.ERROR_CODE["TRAN_MODE_UNSUPPORTED"]: NotSupportedError, + cdb2.ERROR_CODE["VERIFY_ERROR"]: OperationalError, + cdb2.ERROR_CODE["FKEY_VIOLATION"]: ForeignKeyConstraintError, + cdb2.ERROR_CODE["NULL_CONSTRAINT"]: NonNullConstraintError, + cdb2.ERROR_CODE["CONV_FAIL"]: DataError, + cdb2.ERROR_CODE["NONKLESS"]: NotSupportedError, + cdb2.ERROR_CODE["MALLOC"]: OperationalError, + cdb2.ERROR_CODE["NOTSUPPORTED"]: NotSupportedError, + cdb2.ERROR_CODE["DUPLICATE"]: UniqueKeyConstraintError, + cdb2.ERROR_CODE["TZNAME_FAIL"]: DataError, + cdb2.ERROR_CODE["UNKNOWN"]: OperationalError, } def _raise_wrapped_exception(exc): code = exc.error_code - msg = '%s (cdb2api rc %d)' % (exc.error_message, code) + msg = "%s (cdb2api rc %d)" % (exc.error_message, code) if "null constraint violation" in msg: raise NonNullConstraintError(msg) from exc # DRQS 86013831 raise _EXCEPTION_BY_RC.get(code, OperationalError)(msg) from exc @@ -537,7 +565,7 @@ def _sql_operation(sql): def _operation_ends_transaction(operation): - return operation == 'commit' or operation == 'rollback' + return operation == "commit" or operation == "rollback" def _modifies_rows(operation): @@ -545,14 +573,14 @@ def _modifies_rows(operation): # exec is deliberately excluded because it might return a result set, and # this function is used to determine whether it's safe to call # cdb2_get_effects after running the operation. - return operation in ('commit', 'insert', 'update', 'delete') + return operation in ("commit", "insert", "update", "delete") def connect( database_name: str | bytes, - tier: str | bytes="default", - autocommit: bool=False, - host: str | bytes | None=None + tier: str | bytes = "default", + autocommit: bool = False, + host: str | bytes | None = None, ) -> Connection: """Establish a connection to a Comdb2 database. @@ -633,11 +661,18 @@ class Connection: statements, disabling DB-API 2.0's automatic implicit transactions. """ - def __init__(self, database_name: str | bytes, tier: str | bytes="default", autocommit: bool=False, - host: str | bytes | None=None) -> None: + def __init__( + self, + database_name: str | bytes, + tier: str | bytes = "default", + autocommit: bool = False, + host: str | bytes | None = None, + ) -> None: if host is not None and tier != "default": - raise InterfaceError("Connecting to a host by name and to a " - "cluster by tier are mutually exclusive") + raise InterfaceError( + "Connecting to a host by name and to a " + "cluster by tier are mutually exclusive" + ) self._active_cursor = None self._in_transaction = False @@ -674,7 +709,9 @@ def row_factory(self) -> Callable[[list[str]], Callable[[list[Value]], Row]]: return self._hndl.row_factory @row_factory.setter - def row_factory(self, value: Callable[[list[str]], Callable[[list[Value]], Row]]) -> None: + def row_factory( + self, value: Callable[[list[str]], Callable[[list[Value]], Row]] + ) -> None: self._check_closed() self._hndl.row_factory = value @@ -692,7 +729,7 @@ def _execute(self, operation): cursor = self.cursor() cursor._execute(operation, operation) - def close(self, ack_current_event: bool=True) -> None: + def close(self, ack_current_event: bool = True) -> None: """Gracefully close the Comdb2 connection. Once a `Connection` has been closed, no further operations may be @@ -801,9 +838,9 @@ class Cursor: """ _ErrorMessagesByOperation = { - 'begin': "Transactions may not be started explicitly", - 'commit': "Use Connection.commit to commit transactions", - 'rollback': "Use Connection.rollback to roll back transactions", + "begin": "Transactions may not be started explicitly", + "commit": "Use Connection.commit to commit transactions", + "rollback": "Use Connection.rollback to roll back transactions", } def __init__(self, conn: Connection) -> None: @@ -832,7 +869,9 @@ def arraysize(self, value: int) -> None: self._arraysize = value @property - def description(self) -> tuple[tuple[str, object, None, None, None, None, None], ...]: + def description( + self, + ) -> tuple[tuple[str, object, None, None, None, None, None], ...]: """Provides the name and type of each column in the latest result set. This read-only attribute will contain one element per column in the @@ -940,13 +979,19 @@ def callproc(self, procname: str, parameters: Sequence[Value]) -> Sequence[Value if not _VALID_SP_NAME.match(procname): raise NotSupportedError("Invalid procedure name '%s'" % procname) params_as_dict = {str(i): e for i, e in enumerate(parameters)} - sql = ("exec procedure " + procname + "(" - + ", ".join("%%(%d)s" % i for i in range(len(params_as_dict))) - + ")") + sql = ( + "exec procedure " + + procname + + "(" + + ", ".join("%%(%d)s" % i for i in range(len(params_as_dict))) + + ")" + ) self.execute(sql, params_as_dict) return parameters[:] - def execute(self, sql: str, parameters: Mapping[str, Value] | None=None) -> Cursor: + def execute( + self, sql: str, parameters: Mapping[str, Value] | None = None + ) -> Cursor: """Execute a database operation (query or command). The ``sql`` string must be provided as a Python format string, with @@ -998,7 +1043,9 @@ def execute(self, sql: str, parameters: Mapping[str, Value] | None=None) -> Curs # return an iterable over the rows, but this isn't portable across DBs. return self - def executemany(self, sql: str, seq_of_parameters: Sequence[Mapping[str, Value]]) -> None: + def executemany( + self, sql: str, seq_of_parameters: Sequence[Mapping[str, Value]] + ) -> None: """Execute the same SQL statement repeatedly with different parameters. This is currently equivalent to calling execute multiple times, once @@ -1050,7 +1097,7 @@ def _execute(self, operation, sql, parameters=None): except cdb2.Error as e: _raise_wrapped_exception(e) - if operation == 'begin': + if operation == "begin": self._conn._in_transaction = True # txn successfully started elif not self._conn._in_transaction and _modifies_rows(operation): # We're not in a transaction, and the last statement could have @@ -1063,7 +1110,7 @@ def setinputsizes(self, sizes: Sequence[Any]) -> None: """No-op; implemented for PEP-249 compliance.""" self._check_closed() - def setoutputsize(self, size: Any, column: int=None) -> None: + def setoutputsize(self, size: Any, column: int = None) -> None: """No-op; implemented for PEP-249 compliance.""" self._check_closed() @@ -1076,8 +1123,10 @@ def _update_rowcount(self): def _load_description(self): names = self._hndl.column_names() types = self._hndl.column_types() - self._description = tuple((name, type, None, None, None, None, None) - for name, type in zip(names, types)) + self._description = tuple( + (name, type, None, None, None, None, None) + for name, type in zip(names, types) + ) if not self._description: self._description = None @@ -1096,7 +1145,7 @@ def fetchone(self) -> Row | None: except StopIteration: return None - def fetchmany(self, n: int | None=None) -> Sequence[Row]: + def fetchmany(self, n: int | None = None) -> Sequence[Row]: """Fetch the next set of rows of the current result set. Args: diff --git a/comdb2/factories.py b/comdb2/factories.py index fb8ace2..da41058 100644 --- a/comdb2/factories.py +++ b/comdb2/factories.py @@ -79,10 +79,10 @@ def namedtuple_row_factory(col_names: list[str]) -> Callable[[list[Value]], Name # Ensure DML doesn't raise an exception for an invalid column name if len(col_names) == 1: if col_names[0] in ("rows inserted", "rows updated", "rows deleted"): - return namedtuple('Row', col_names, rename=True)._make + return namedtuple("Row", col_names, rename=True)._make try: - return namedtuple('Row', col_names)._make + return namedtuple("Row", col_names)._make except ValueError: # If the error was caused by duplicated column names, raise a more # preceise error message. Otherwise, re-raise. @@ -122,8 +122,10 @@ def dict_row_factory(col_names: list[str]) -> Callable[[list[Value]], dict[str, 1 """ _raise_on_duplicate_column_names(col_names) + def dict_row(col_vals): return dict(zip(col_names, col_vals)) + return dict_row diff --git a/docs/conf.py b/docs/conf.py index 8e2f22b..0c57c46 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,59 +13,60 @@ import sys import os -sys.path.insert(0, os.path.abspath('..')) + +sys.path.insert(0, os.path.abspath("..")) from comdb2 import __version__ # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.intersphinx', - 'sphinx.ext.napoleon', - 'sphinx.ext.viewcode', - 'sphinx.ext.githubpages', + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", + "sphinx.ext.napoleon", + "sphinx.ext.viewcode", + "sphinx.ext.githubpages", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['templates'] +templates_path = ["templates"] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The root toctree document. -root_doc = 'index' +root_doc = "index" # General information about the project. -project = 'Comdb2' -copyright = '2017, Bloomberg LP' +project = "Comdb2" +copyright = "2017, Bloomberg LP" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # -version = '.'.join(__version__.split('.')[:2]) +version = ".".join(__version__.split(".")[:2]) # The full version, including alpha/beta/rc tags. release = __version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -73,58 +74,58 @@ # The reST default role (used for this markup: `text`) to use for all # documents. -default_role = 'py:obj' +default_role = "py:obj" # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. -modindex_common_prefix = ['comdb2.'] +modindex_common_prefix = ["comdb2."] # If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False +# keep_warnings = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, @@ -134,51 +135,51 @@ # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +# html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'Comdb2doc' +htmlhelp_basename = "Comdb2doc" # -- Options for LaTeX output --------------------------------------------- @@ -204,23 +205,23 @@ # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output --------------------------------------- @@ -233,7 +234,7 @@ # ] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------- @@ -248,26 +249,26 @@ # ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +# texinfo_no_detailmenu = False # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} +intersphinx_mapping = {"python": ("https://docs.python.org/3", None)} # Don't highlight the code for the Python standard library datetime module. viewcode_import = False # Don't automatically infer types from type annotations. -autodoc_typehints = 'none' +autodoc_typehints = "none" # Replace the extension module with a mock. -autodoc_mock_imports = ['comdb2._ccdb2'] +autodoc_mock_imports = ["comdb2._ccdb2"] diff --git a/setup.py b/setup.py index 63d9b94..432e302 100644 --- a/setup.py +++ b/setup.py @@ -15,39 +15,39 @@ from setuptools import setup, Extension about = {} -with open('comdb2/_about.py') as fp: +with open("comdb2/_about.py") as fp: exec(fp.read(), about) def make_static_extension(name, **kwargs): - libraries = kwargs.pop('libraries', []) - pkg_config = pkgconfig.parse(' '.join(libraries), static=True) - libraries = pkg_config['libraries'] - library_dirs = pkg_config['library_dirs'] - include_dirs = kwargs.pop("include_dirs", []) + pkg_config['include_dirs'] + libraries = kwargs.pop("libraries", []) + pkg_config = pkgconfig.parse(" ".join(libraries), static=True) + libraries = pkg_config["libraries"] + library_dirs = pkg_config["library_dirs"] + include_dirs = kwargs.pop("include_dirs", []) + pkg_config["include_dirs"] return Extension( name, libraries=libraries, library_dirs=library_dirs, include_dirs=include_dirs, - **kwargs + **kwargs, ) ccdb2 = make_static_extension( "comdb2._ccdb2", - extra_compile_args=['-std=c99'], - libraries=['cdb2api'], - sources=["comdb2/_ccdb2.pyx"] + extra_compile_args=["-std=c99"], + libraries=["cdb2api"], + sources=["comdb2/_ccdb2.pyx"], ) setup( - name='comdb2', + name="comdb2", version=about["__version__"], - author='Alex Chamberlain', - author_email='achamberlai9@bloomberg.net', - packages=['comdb2'], + author="Alex Chamberlain", + author_email="achamberlai9@bloomberg.net", + packages=["comdb2"], install_requires=["pytz"], extras_require={"tests": ["python-dateutil>=2.6.0", "pytest"]}, python_requires=">=3.7", diff --git a/tests/test_cdb2.py b/tests/test_cdb2.py index 381bc9a..d9c0203 100644 --- a/tests/test_cdb2.py +++ b/tests/test_cdb2.py @@ -15,15 +15,16 @@ from comdb2.factories import namedtuple_row_factory import pytest -COLUMN_LIST = ("short_col u_short_col int_col u_int_col longlong_col" - " float_col double_col byte_col byte_array_col" - " cstring_col pstring_col blob_col datetime_col vutf8_col" - ).split() +COLUMN_LIST = ( + "short_col u_short_col int_col u_int_col longlong_col" + " float_col double_col byte_col byte_array_col" + " cstring_col pstring_col blob_col datetime_col vutf8_col" +).split() @pytest.fixture(autouse=True) def delete_all_rows(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") while True: hndl.execute("delete from all_datatypes limit 100") if hndl.get_effects()[0] != 100: @@ -36,17 +37,17 @@ def delete_all_rows(): def test_garbage_collecting_unused_handle(): - cdb2.Handle('mattdb', 'dev').execute("select 1 union select 2") + cdb2.Handle("mattdb", "dev").execute("select 1 union select 2") def test_commit_on_unused_connection(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") with pytest.raises(cdb2.Error): hndl.execute("commit") def test_empty_transactions(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") hndl.execute("begin") hndl.execute("commit") @@ -55,7 +56,7 @@ def test_empty_transactions(): def test_binding_parameters(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") hndl.execute("insert into simple(key, val) values(@k, @v)", dict(v=2, k=1)) hndl.execute("insert into simple(key, val) values(@k, @v)", dict(k=3, v=4)) assert hndl.get_effects()[0] == 1 @@ -65,7 +66,7 @@ def test_binding_parameters(): def test_commit_failures(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") hndl.execute("begin") hndl.execute("insert into simple(key, val) values(@k, @v)", dict(k=1, v=2)) hndl.execute("insert into simple(key, val) values(@k, @v)", dict(k=3)) @@ -73,78 +74,78 @@ def test_commit_failures(): try: hndl.execute("commit") except cdb2.Error as exc: - assert exc.error_code == cdb2.ERROR_CODE['PREPARE_ERROR'] + assert exc.error_code == cdb2.ERROR_CODE["PREPARE_ERROR"] else: assert False # an assertion should have been raised! def test_error_from_closing_connection_twice(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") hndl.close() with pytest.raises(cdb2.Error): hndl.close() def test_iterating_newly_initialized_handle(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") for row in hndl: assert False def test_timezone_handling(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") rows = list(hndl.execute("select now()")) assert len(rows) == 1 - assert rows[0][0].tzname() == 'UTC' + assert rows[0][0].tzname() == "UTC" - hndl = cdb2.Handle('mattdb', 'dev', tz='GMT') + hndl = cdb2.Handle("mattdb", "dev", tz="GMT") rows = list(hndl.execute("select now()")) assert len(rows) == 1 - assert rows[0][0].tzname() == 'GMT' + assert rows[0][0].tzname() == "GMT" hndl.execute("set timezone UTC") rows = list(hndl.execute("select now()")) assert len(rows) == 1 - assert rows[0][0].tzname() == 'UTC' + assert rows[0][0].tzname() == "UTC" def test_passing_flags(): # Bad SQL statements in a transaction fail early - flags = cdb2.HANDLE_FLAGS['READ_INTRANS_RESULTS'] + flags = cdb2.HANDLE_FLAGS["READ_INTRANS_RESULTS"] # Test passing flags - hndl = cdb2.Handle('mattdb', 'dev', flags=flags) - hndl.execute('begin') + hndl = cdb2.Handle("mattdb", "dev", flags=flags) + hndl.execute("begin") with pytest.raises(cdb2.Error): - hndl.execute('foobar') + hndl.execute("foobar") # Default behavior, without any flags passed - hndl = cdb2.Handle('mattdb', 'dev', flags=0) - hndl.execute('begin') - hndl.execute('foobar') + hndl = cdb2.Handle("mattdb", "dev", flags=0) + hndl.execute("begin") + hndl.execute("foobar") with pytest.raises(cdb2.Error): hndl.execute("select 1") def test_row_factories(): query = "select 1 as 'a', 2 as 'b' union select 3, 4 order by a" - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") assert list(hndl.execute(query)) == [[1, 2], [3, 4]] hndl.row_factory = dict_row_factory assert hndl.row_factory == dict_row_factory - assert list(hndl.execute(query)) == [{'a': 1, 'b': 2}, {'a': 3, 'b': 4}] + assert list(hndl.execute(query)) == [{"a": 1, "b": 2}, {"a": 3, "b": 4}] hndl.row_factory = namedtuple_row_factory assert hndl.row_factory == namedtuple_row_factory rows = [r._asdict() for r in hndl.execute(query)] - assert rows == [{'a': 1, 'b': 2}, {'a': 3, 'b': 4}] + assert rows == [{"a": 1, "b": 2}, {"a": 3, "b": 4}] def test_row_factories_with_dup_col_names(): query = "select 1 as 'a', 2 as 'a'" - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") assert list(hndl.execute(query)) == [[1, 2]] @@ -160,19 +161,19 @@ def test_row_factories_with_dup_col_names(): def test_setting_non_callable_row_factory(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") with pytest.raises(TypeError): hndl.row_factory = "foo" def test_unsetting_row_factory(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") assert list(hndl.execute("select 1")) == [[1]] hndl.row_factory = dict_row_factory - assert list(hndl.execute("select 1")) == [{'1': 1}] + assert list(hndl.execute("select 1")) == [{"1": 1}] hndl.row_factory = None assert list(hndl.execute("select 1")) == [[1]] @@ -182,11 +183,12 @@ def test_failures_instantiating_row_class(): def factory(col_names): def klass(col_values): raise TypeError(col_values) + return klass query = "select 1 as 'a', 2 as 'a'" - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") hndl.row_factory = factory it = iter(hndl.execute(query)) with pytest.raises(cdb2.Error) as exc_info: @@ -195,7 +197,7 @@ def klass(col_values): def test_get_effects(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") hndl.execute("select 1 union all select 2") assert hndl.get_effects() == (0, 2, 0, 0, 0) @@ -220,45 +222,46 @@ def test_get_effects(): @pytest.mark.xfail(reason="cdb2_open ignores tier") def test_nonascii_error_messages(): with pytest.raises(cdb2.Error) as exc_info: - cdb2.Handle('mattdb', b'\xc3') - assert r'cluster type \xc3.' in exc_info.value.error_message + cdb2.Handle("mattdb", b"\xc3") + assert r"cluster type \xc3." in exc_info.value.error_message - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") with pytest.raises(cdb2.Error) as exc_info: - hndl.execute(b'select \xC3') - assert r'no such column: \xc3' in exc_info.value.error_message + hndl.execute(b"select \xc3") + assert r"no such column: \xc3" in exc_info.value.error_message teststr = bytes(bytearray(range(128, 256))) with pytest.raises(cdb2.Error) as exc_info: - hndl.execute(b'select ' + teststr) + hndl.execute(b"select " + teststr) assert ( - r'no such column: ' - r'\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f' - r'\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f' - r'\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf' - r'\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf' - r'\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf' - r'\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf' - r'\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef' - r'\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff' - in exc_info.value.error_message) + r"no such column: " + r"\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f" + r"\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f" + r"\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf" + r"\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf" + r"\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf" + r"\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf" + r"\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef" + r"\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" + in exc_info.value.error_message + ) teststr = bytes(bytearray(range(1, 31))) with pytest.raises(cdb2.Error) as exc_info: - cdb2.Handle('mattdb', teststr) - errmsg_utf8 = exc_info.value.error_message.encode('utf-8') + cdb2.Handle("mattdb", teststr) + errmsg_utf8 = exc_info.value.error_message.encode("utf-8") assert teststr in errmsg_utf8 - teststr = bytes(bytearray(range(1, 31))) + b'\xff' + teststr = bytes(bytearray(range(1, 31))) + b"\xff" with pytest.raises(cdb2.Error) as exc_info: - cdb2.Handle('mattdb', teststr) - errmsg_utf8 = exc_info.value.error_message.encode('utf-8') + cdb2.Handle("mattdb", teststr) + errmsg_utf8 = exc_info.value.error_message.encode("utf-8") assert teststr not in errmsg_utf8 - assert teststr.decode('latin1').encode('unicode_escape') in errmsg_utf8 + assert teststr.decode("latin1").encode("unicode_escape") in errmsg_utf8 def test_namedtuple_factory_dml(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") hndl.row_factory = namedtuple_row_factory hndl.execute("insert into simple (key, val) values (1, 1)") @@ -272,7 +275,7 @@ def test_namedtuple_factory_dml(): def test_parameter_name_in_binding_errors_exception(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") class BadBytes(bytes): def __len__(self): @@ -290,10 +293,10 @@ def __repr__(self): def test_parameter_name_in_binding_errors_noexception(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") with pytest.raises(Exception) as exc: - hndl.execute("select @param", dict(param=(5+6j))) + hndl.execute("select @param", dict(param=(5 + 6j))) assert exc.value.args[1] == ( "Can't map complex value (5+6j) for parameter 'param' to a Comdb2 type" diff --git a/tests/test_cdb2_datetimeus.py b/tests/test_cdb2_datetimeus.py index 7266ba0..3589722 100644 --- a/tests/test_cdb2_datetimeus.py +++ b/tests/test_cdb2_datetimeus.py @@ -84,7 +84,7 @@ def test_datetimeus_fromtimestamp(): def test_datetimeus_astimezone(): - eastern = pytz.timezone('US/Eastern') + eastern = pytz.timezone("US/Eastern") loc_dt = eastern.localize(datetime.datetime(2002, 10, 27, 6, 0, 0)) dtu = cdb2.DatetimeUs.fromdatetime(loc_dt) @@ -95,7 +95,7 @@ def test_datetimeus_type_stickiness(): def check(obj): assert isinstance(obj, cdb2.DatetimeUs) - new_york = pytz.timezone('America/New_York') + new_york = pytz.timezone("America/New_York") utc = pytz.UTC check(new_york.localize(cdb2.DatetimeUs(2016, 8, 15, 18, 47, 15, 123456))) @@ -119,16 +119,17 @@ def check(obj): check(cdb2.DatetimeUs.now(utc).astimezone(new_york)) -@pytest.mark.skipif(not hasattr(datetime.datetime, 'fold'), - reason='Skipped before PEP 495') +@pytest.mark.skipif( + not hasattr(datetime.datetime, "fold"), reason="Skipped before PEP 495" +) def test_datetimeus_fold(): - NYC = dateutil.tz.gettz('America/New_York') + NYC = dateutil.tz.gettz("America/New_York") dt = datetime.datetime(2004, 10, 31, 1, 30, fold=1, tzinfo=NYC) dtus = cdb2.DatetimeUs.fromdatetime(dt) - assert dtus.fold == 1 # Check that it handles fold correctly + assert dtus.fold == 1 # Check that it handles fold correctly # Make sure that the time is correctly disambiguating - assert dt.tzname() == "EST" # Just in case it's not a DatetimeUs problem + assert dt.tzname() == "EST" # Just in case it's not a DatetimeUs problem assert dtus.tzname() == "EST" diff --git a/tests/test_dbapi2.py b/tests/test_dbapi2.py index 25409c4..db51f53 100644 --- a/tests/test_dbapi2.py +++ b/tests/test_dbapi2.py @@ -41,19 +41,33 @@ from unittest.mock import patch -COLUMN_LIST = ("short_col u_short_col int_col u_int_col longlong_col" - " float_col double_col byte_col byte_array_col" - " cstring_col pstring_col blob_col datetime_col vutf8_col" - ).split() - -COLUMN_TYPE = (NUMBER, NUMBER, NUMBER, NUMBER, NUMBER, - NUMBER, NUMBER, BINARY, BINARY, - STRING, STRING, BINARY, DATETIME, STRING) +COLUMN_LIST = ( + "short_col u_short_col int_col u_int_col longlong_col" + " float_col double_col byte_col byte_array_col" + " cstring_col pstring_col blob_col datetime_col vutf8_col" +).split() + +COLUMN_TYPE = ( + NUMBER, + NUMBER, + NUMBER, + NUMBER, + NUMBER, + NUMBER, + NUMBER, + BINARY, + BINARY, + STRING, + STRING, + BINARY, + DATETIME, + STRING, +) @pytest.fixture(autouse=True) def delete_all_rows(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() while True: cursor.execute("delete from all_datatypes limit 100") @@ -76,37 +90,37 @@ def delete_all_rows(): @pytest.mark.xfail(reason="cdb2_open ignores tier") def test_invalid_cluster(): with pytest.raises(OperationalError): - connect('mattdb', 'foo') + connect("mattdb", "foo") def test_invalid_dbname(): with pytest.raises(OperationalError): - connect('', 'dev') + connect("", "dev") def test_closing_unused_connection(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") conn.close() def test_garbage_collecting_unused_connection(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") del conn def test_commit_on_unused_connection(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") conn.commit() def test_rollback_on_unused_connection(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") conn.rollback() def test_inserts(): # Without params - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("insert into simple(key, val) values(1, 2)") cursor.connection.commit() @@ -116,10 +130,9 @@ def test_inserts(): assert cursor.fetchall() == [[1, 2]] # With params - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() - cursor.execute("insert into simple(key, val) values(%(k)s, %(v)s)", - dict(k=3, v=4)) + cursor.execute("insert into simple(key, val) values(%(k)s, %(v)s)", dict(k=3, v=4)) conn.commit() assert cursor.rowcount == 1 @@ -128,7 +141,7 @@ def test_inserts(): def test_rollback(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("insert into simple(key, val) values(1, 2)") conn.rollback() @@ -138,10 +151,12 @@ def test_rollback(): def test_commit_failures(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() - cursor.executemany("insert into simple(key, val) values(%(key)s, %(val)s)", - [ dict(key=1, val=2), dict(key=3, val=None) ]) + cursor.executemany( + "insert into simple(key, val) values(%(key)s, %(val)s)", + [dict(key=1, val=2), dict(key=3, val=None)], + ) with pytest.raises(IntegrityError): conn.commit() assert cursor.rowcount == 0 @@ -151,7 +166,7 @@ def test_commit_failures(): def test_unique_key_violation(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("insert into simple(key, val) values(1, 2)") conn.commit() @@ -161,42 +176,43 @@ def test_unique_key_violation(): def test_constraint_errors(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("insert into simple(key, val) values(1, 2)") cursor.execute("insert into simple(key, val) values(1, 2)") with pytest.raises(UniqueKeyConstraintError) as exc_info: conn.commit() - errcode = ' (cdb2api rc %d)' % cdb2.ERROR_CODE['DUPLICATE'] + errcode = " (cdb2api rc %d)" % cdb2.ERROR_CODE["DUPLICATE"] assert errcode in str(exc_info.value) cursor.execute("insert into simple(key, val) values(null, 2)") with pytest.raises(NonNullConstraintError) as exc_info: conn.commit() - errcode = ' (cdb2api rc %d)' % cdb2.ERROR_CODE['NULL_CONSTRAINT'] + errcode = " (cdb2api rc %d)" % cdb2.ERROR_CODE["NULL_CONSTRAINT"] assert errcode in str(exc_info.value) cursor.execute("insert into simple(key, val) values(1, 2)") conn.commit() cursor.execute("selectv * from simple") - connect('mattdb', 'dev', autocommit=True).cursor().execute( - "update simple set key=2") + connect("mattdb", "dev", autocommit=True).cursor().execute( + "update simple set key=2" + ) with pytest.raises(IntegrityError) as exc_info: conn.commit() - errcode = ' (cdb2api rc %d)' % cdb2.ERROR_CODE['CONSTRAINTS'] + errcode = " (cdb2api rc %d)" % cdb2.ERROR_CODE["CONSTRAINTS"] assert errcode in str(exc_info.value) cursor.execute("insert into child(key) values(1)") with pytest.raises(ForeignKeyConstraintError) as exc_info: conn.commit() - errcode = ' (cdb2api rc %d)' % cdb2.ERROR_CODE['FKEY_VIOLATION'] + errcode = " (cdb2api rc %d)" % cdb2.ERROR_CODE["FKEY_VIOLATION"] assert errcode in str(exc_info.value) def test_implicitly_closing_old_cursor_when_opening_a_new_one(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor1 = conn.cursor() cursor2 = conn.cursor() with pytest.raises(InterfaceError): @@ -205,7 +221,7 @@ def test_implicitly_closing_old_cursor_when_opening_a_new_one(): def test_commit_after_cursor_close(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("insert into simple(key, val) values(1, 2)") conn.commit() @@ -219,40 +235,39 @@ def test_commit_after_cursor_close(): def test_implicit_rollback_on_connection_close(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("insert into simple(key, val) values(1, 2)") conn.commit() cursor.execute("insert into simple(key, val) values(3, 4)") - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() rows = list(cursor.execute("select key, val from simple order by key")) assert rows == [[1, 2]] def test_extra_percent_arg(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() with pytest.raises(InterfaceError): - cursor.execute("insert into simple(key, val) values(%(k)s, %(v)s)", - dict(k=3)) + cursor.execute("insert into simple(key, val) values(%(k)s, %(v)s)", dict(k=3)) def test_unescaped_percent(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() - cursor.execute("select 1%%2" ) # Should work + cursor.execute("select 1%%2") # Should work with pytest.raises(InterfaceError): cursor.execute("select 1%2") def test_reading_and_writing_datetimes(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() ts_obj = Timestamp(2015, 1, 2, 3, 4, 5, 123000, pytz.UTC) - ts_str_in = '2015-01-02T03:04:05.12345' - ts_str_out = '2015-01-02T030405.123 UTC' + ts_str_in = "2015-01-02T03:04:05.12345" + ts_str_out = "2015-01-02T030405.123 UTC" cursor.execute("select cast(%(x)s as date)", dict(x=ts_str_in)) assert cursor.fetchall() == [[ts_obj]] @@ -262,11 +277,11 @@ def test_reading_and_writing_datetimes(): def test_reading_and_writing_datetimeus(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() ts_obj = TimestampUs(2015, 1, 2, 3, 4, 5, 123456, pytz.UTC) - ts_str_in = '2015-01-02T03:04:05.123456' - ts_str_out = '2015-01-02T030405.123456 UTC' + ts_str_in = "2015-01-02T03:04:05.123456" + ts_str_out = "2015-01-02T030405.123456 UTC" cursor.execute("select cast(%(x)s as date)", dict(x=ts_str_in)) assert cursor.fetchall() == [[ts_obj]] @@ -276,37 +291,51 @@ def test_reading_and_writing_datetimeus(): def test_inserting_one_row_with_all_datatypes_without_parameters(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() - cursor.execute("insert into all_datatypes(" + ', '.join(COLUMN_LIST) + ")" - " values(1, 2, 3, 4, 5, .5, .25, x'01', x'0102030405'," - " 'hello', 'goodbye', x'01020304050607'," - " cast(1234567890.2345 as datetime), 'hello world')") + cursor.execute( + "insert into all_datatypes(" + ", ".join(COLUMN_LIST) + ")" + " values(1, 2, 3, 4, 5, .5, .25, x'01', x'0102030405'," + " 'hello', 'goodbye', x'01020304050607'," + " cast(1234567890.2345 as datetime), 'hello world')" + ) conn.commit() assert cursor.rowcount == 1 - cursor.execute("select " + ', '.join(COLUMN_LIST) + " from all_datatypes") + cursor.execute("select " + ", ".join(COLUMN_LIST) + " from all_datatypes") for i in range(len(COLUMN_LIST)): assert cursor.description[i][0] == COLUMN_LIST[i] for type_object in (STRING, BINARY, NUMBER, DATETIME): - assert ((type_object == cursor.description[i][1]) - == (type_object == COLUMN_TYPE[i])) + assert (type_object == cursor.description[i][1]) == ( + type_object == COLUMN_TYPE[i] + ) assert cursor.description[i][2:] == (None, None, None, None, None) row = cursor.fetchone() - assert row == [1, 2, 3, 4, 5, 0.5, 0.25, - Binary('\x01'), Binary('\x01\x02\x03\x04\x05'), - 'hello', 'goodbye', - Binary('\x01\x02\x03\x04\x05\x06\x07'), - pytz.timezone("America/New_York").localize( - Datetime(2009, 2, 13, 18, 31, 30, 234000)), - "hello world"] + assert row == [ + 1, + 2, + 3, + 4, + 5, + 0.5, + 0.25, + Binary("\x01"), + Binary("\x01\x02\x03\x04\x05"), + "hello", + "goodbye", + Binary("\x01\x02\x03\x04\x05\x06\x07"), + pytz.timezone("America/New_York").localize( + Datetime(2009, 2, 13, 18, 31, 30, 234000) + ), + "hello world", + ] assert cursor.fetchone() is None def test_all_datatypes_as_parameters(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() params = ( ("short_col", 32767), @@ -314,23 +343,30 @@ def test_all_datatypes_as_parameters(): ("int_col", 2147483647), ("u_int_col", 4294967295), ("longlong_col", 9223372036854775807), - ("float_col", .125), - ("double_col", 2.**65), - ("byte_col", Binary(b'\x00')), - ("byte_array_col", Binary(b'\x02\x01\x00\x01\x02')), - ("cstring_col", 'HELLO'), - ("pstring_col", 'GOODBYE'), - ("blob_col", Binary('')), - ("datetime_col", pytz.timezone("America/New_York").localize( - Datetime(2009, 2, 13, 18, 31, 30, 234000))), - ("vutf8_col", "foo" * 50) + ("float_col", 0.125), + ("double_col", 2.0**65), + ("byte_col", Binary(b"\x00")), + ("byte_array_col", Binary(b"\x02\x01\x00\x01\x02")), + ("cstring_col", "HELLO"), + ("pstring_col", "GOODBYE"), + ("blob_col", Binary("")), + ( + "datetime_col", + pytz.timezone("America/New_York").localize( + Datetime(2009, 2, 13, 18, 31, 30, 234000) + ), + ), + ("vutf8_col", "foo" * 50), + ) + cursor.execute( + "insert into all_datatypes(" + ", ".join(COLUMN_LIST) + ")" + " values(%(short_col)s, %(u_short_col)s, %(int_col)s," + " %(u_int_col)s, %(longlong_col)s, %(float_col)s," + " %(double_col)s, %(byte_col)s, %(byte_array_col)s," + " %(cstring_col)s, %(pstring_col)s, %(blob_col)s," + " %(datetime_col)s, %(vutf8_col)s)", + dict(params), ) - cursor.execute("insert into all_datatypes(" + ', '.join(COLUMN_LIST) + ")" - " values(%(short_col)s, %(u_short_col)s, %(int_col)s," - " %(u_int_col)s, %(longlong_col)s, %(float_col)s," - " %(double_col)s, %(byte_col)s, %(byte_array_col)s," - " %(cstring_col)s, %(pstring_col)s, %(blob_col)s," - " %(datetime_col)s, %(vutf8_col)s)", dict(params)) conn.commit() @@ -341,7 +377,7 @@ def test_all_datatypes_as_parameters(): def test_naive_datetime_as_parameter(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() params = ( ("short_col", 32767), @@ -349,23 +385,26 @@ def test_naive_datetime_as_parameter(): ("int_col", 2147483647), ("u_int_col", 4294967295), ("longlong_col", 9223372036854775807), - ("float_col", .125), - ("double_col", 2.**65), - ("byte_col", Binary(b'\x00')), - ("byte_array_col", Binary(b'\x02\x01\x00\x01\x02')), - ("cstring_col", 'HELLO'), - ("pstring_col", 'GOODBYE'), - ("blob_col", Binary('')), + ("float_col", 0.125), + ("double_col", 2.0**65), + ("byte_col", Binary(b"\x00")), + ("byte_array_col", Binary(b"\x02\x01\x00\x01\x02")), + ("cstring_col", "HELLO"), + ("pstring_col", "GOODBYE"), + ("blob_col", Binary("")), ("datetime_col", Datetime(2009, 2, 13, 18, 31, 30, 234000)), - ("vutf8_col", "foo" * 50) + ("vutf8_col", "foo" * 50), ) - cursor.execute("insert into all_datatypes(" + ', '.join(COLUMN_LIST) + ")" - " values(%(short_col)s, %(u_short_col)s, %(int_col)s," - " %(u_int_col)s, %(longlong_col)s, %(float_col)s," - " %(double_col)s, %(byte_col)s, %(byte_array_col)s," - " %(cstring_col)s, %(pstring_col)s, %(blob_col)s," - " %(datetime_col)s, %(vutf8_col)s)", dict(params)) + cursor.execute( + "insert into all_datatypes(" + ", ".join(COLUMN_LIST) + ")" + " values(%(short_col)s, %(u_short_col)s, %(int_col)s," + " %(u_int_col)s, %(longlong_col)s, %(float_col)s," + " %(double_col)s, %(byte_col)s, %(byte_array_col)s," + " %(cstring_col)s, %(pstring_col)s, %(blob_col)s," + " %(datetime_col)s, %(vutf8_col)s)", + dict(params), + ) cursor.connection.commit() cursor.execute("select datetime_col from all_datatypes") @@ -375,40 +414,40 @@ def test_naive_datetime_as_parameter(): def test_datetime_with_non_olson_tzname(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() - tz = pytz.timezone('America/New_York') + tz = pytz.timezone("America/New_York") dt = datetime.datetime(2016, 11, 6, 1, 30, 0, 123000) est_dt = tz.localize(dt, is_dst=False) edt_dt = tz.localize(dt, is_dst=True) - assert est_dt.tzname() == 'EST' - assert edt_dt.tzname() == 'EDT' - params = {'est_dt': est_dt, 'edt_dt': edt_dt} + assert est_dt.tzname() == "EST" + assert edt_dt.tzname() == "EDT" + params = {"est_dt": est_dt, "edt_dt": edt_dt} row = cursor.execute("select @est_dt, @edt_dt", params).fetchone() - assert row[0].tzname() == 'UTC' + assert row[0].tzname() == "UTC" assert row[0] == est_dt - assert row[1].tzname() == 'UTC' + assert row[1].tzname() == "UTC" assert row[1] == edt_dt def test_rounding_datetime_to_nearest_millisecond(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() curr_microsecond = Datetime(2016, 2, 28, 23, 59, 59, 999499, pytz.UTC) prev_millisecond = Datetime(2016, 2, 28, 23, 59, 59, 999000, pytz.UTC) next_millisecond = Datetime(2016, 2, 29, 0, 0, 0, 0, pytz.UTC) - cursor.execute("select @date", {'date': curr_microsecond}) + cursor.execute("select @date", {"date": curr_microsecond}) assert cursor.fetchall() == [[prev_millisecond]] curr_microsecond += datetime.timedelta(microseconds=1) - cursor.execute("select @date", {'date': curr_microsecond}) + cursor.execute("select @date", {"date": curr_microsecond}) assert cursor.fetchall() == [[next_millisecond]] def test_cursor_description(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() assert cursor.description is None @@ -422,12 +461,14 @@ def test_cursor_description(): cursor.execute("select '1' as foo, cast(1 as datetime) bar") assert cursor.description == ( ("foo", STRING, None, None, None, None, None), - ("bar", DATETIME, None, None, None, None, None)) + ("bar", DATETIME, None, None, None, None, None), + ) cursor.connection.commit() assert cursor.description == ( ("foo", STRING, None, None, None, None, None), - ("bar", DATETIME, None, None, None, None, None)) + ("bar", DATETIME, None, None, None, None, None), + ) cursor.execute("insert into simple(key, val) values(3, 4)") assert cursor.description is None @@ -438,12 +479,14 @@ def test_cursor_description(): cursor.execute("select key, val from simple") assert cursor.description == ( ("key", NUMBER, None, None, None, None, None), - ("val", NUMBER, None, None, None, None, None)) + ("val", NUMBER, None, None, None, None, None), + ) cursor.connection.rollback() assert cursor.description == ( ("key", NUMBER, None, None, None, None, None), - ("val", NUMBER, None, None, None, None, None)) + ("val", NUMBER, None, None, None, None, None), + ) with pytest.raises(ProgrammingError): cursor.execute("select") @@ -451,21 +494,21 @@ def test_cursor_description(): def test_binding_number_that_overflows_long_long(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() with pytest.raises(DataError): cursor.execute("select @i", dict(i=2**64 + 1)) def test_retrieving_null(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("select null, null") assert cursor.fetchall() == [[None, None]] def test_retrieving_interval(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("select cast(0 as datetime) - cast(0 as datetime)") with pytest.raises(NotSupportedError): @@ -473,7 +516,7 @@ def test_retrieving_interval(): def test_syntax_error(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("foo") with pytest.raises(ProgrammingError): @@ -481,21 +524,21 @@ def test_syntax_error(): def test_errors_being_swallowed_during_cursor_close(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("foo") cursor.close() def test_errors_being_swallowed_during_connection_close(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("foo") conn.close() def test_public_connection_methods_after_close(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") conn.close() with pytest.raises(InterfaceError): conn.close() @@ -513,45 +556,45 @@ def test_public_connection_methods_after_close(): def test_misusing_cursor_objects(): with pytest.raises(InterfaceError): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute(" BEGIN TRANSACTION ") with pytest.raises(InterfaceError): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("commit") with pytest.raises(InterfaceError): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute(" rollback ") with pytest.raises(InterfaceError): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.fetchone() with pytest.raises(InterfaceError): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.fetchmany() with pytest.raises(InterfaceError): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.fetchall() def test_noop_methods(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.setinputsizes([1, 2, 3]) cursor.setoutputsize(42) def test_fetchmany(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("select 1 UNION select 2 UNION select 3 order by 1") @@ -573,18 +616,20 @@ def test_fetchmany(): def test_consuming_result_sets_automatically(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("select 1 UNION select 2 UNION select 3 order by 1") cursor.execute("select 1 UNION select 2 UNION select 3 order by 1") def test_inserting_non_utf8_string(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() - cursor.execute("insert into strings values(cast(%(x)s as text), %(y)s)", - dict(x=b'\x68\xeb\x6c\x6c\x6f', y=b'\x68\xeb\x6c\x6c\x6f')) + cursor.execute( + "insert into strings values(cast(%(x)s as text), %(y)s)", + dict(x=b"\x68\xeb\x6c\x6c\x6f", y=b"\x68\xeb\x6c\x6c\x6f"), + ) conn.commit() with pytest.raises(DataError): @@ -592,11 +637,11 @@ def test_inserting_non_utf8_string(): rows = list(cursor) rows = list(cursor.execute("select cast(foo as blob), bar from strings")) - assert rows == [[b'\x68\xeb\x6c\x6c\x6f', b'\x68\xeb\x6c\x6c\x6f']] + assert rows == [[b"\x68\xeb\x6c\x6c\x6f", b"\x68\xeb\x6c\x6c\x6f"]] def test_cursor_connection_attribute_keeps_connection_alive(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() del conn cursor.execute("insert into simple(key, val) values(1, 2)") @@ -608,7 +653,7 @@ def test_cursor_connection_attribute_keeps_connection_alive(): def test_exceptions_containing_unicode_error_messages(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() with pytest.raises(ProgrammingError): try: @@ -620,25 +665,25 @@ def test_exceptions_containing_unicode_error_messages(): def throw_on(expected_stmt, stmt, parameters=None): if stmt == expected_stmt: - raise cdb2.Error(42, 'Not supported error') + raise cdb2.Error(42, "Not supported error") -@patch('comdb2.cdb2.Handle') +@patch("comdb2.cdb2.Handle") def test_begin_throws_error(handle): - handle.return_value.execute.side_effect = partial(throw_on, 'begin') + handle.return_value.execute.side_effect = partial(throw_on, "begin") - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() with pytest.raises(OperationalError): cursor.execute("insert into simple(key, val) values(1, 2)") -@patch('comdb2.cdb2.Handle') +@patch("comdb2.cdb2.Handle") def test_commit_throws_error(handle): - handle.return_value.execute.side_effect = partial(throw_on, 'commit') + handle.return_value.execute.side_effect = partial(throw_on, "commit") - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("insert into simple(key, val) values(1, 2)") @@ -646,21 +691,21 @@ def test_commit_throws_error(handle): conn.commit() -@patch('comdb2.cdb2.Handle') +@patch("comdb2.cdb2.Handle") def test_get_effect_throws_error(handle): def raise_not_supported_error(): - raise cdb2.Error(42, 'Not supported error') + raise cdb2.Error(42, "Not supported error") handle.return_value.get_effects.side_effect = raise_not_supported_error - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("insert into simple(key, val) values(1, 2)") conn.commit() def test_autocommit_handles(): - conn = connect('mattdb', 'dev', autocommit=True) + conn = connect("mattdb", "dev", autocommit=True) cursor = conn.cursor() # Explicit transactions must work @@ -708,24 +753,24 @@ def test_autocommit_handles(): def test_row_factories(): query = "select 1 as 'a', 2 as 'b' union select 3, 4 order by a" - hndl = connect('mattdb', 'dev') + hndl = connect("mattdb", "dev") assert list(hndl.cursor().execute(query)) == [[1, 2], [3, 4]] hndl.row_factory = dict_row_factory assert hndl.row_factory == dict_row_factory rows = list(hndl.cursor().execute(query)) - assert rows == [{'a': 1, 'b': 2}, {'a': 3, 'b': 4}] + assert rows == [{"a": 1, "b": 2}, {"a": 3, "b": 4}] hndl.row_factory = namedtuple_row_factory assert hndl.row_factory == namedtuple_row_factory rows = [r._asdict() for r in hndl.cursor().execute(query)] - assert rows == [{'a': 1, 'b': 2}, {'a': 3, 'b': 4}] + assert rows == [{"a": 1, "b": 2}, {"a": 3, "b": 4}] def test_row_factories_with_dup_col_names(): query = "select 1 as 'a', 2 as 'a', 3 as 'b', 4 as 'b', 5 as 'c'" - hndl = connect('mattdb', 'dev') + hndl = connect("mattdb", "dev") assert list(hndl.cursor().execute(query)) == [[1, 2, 3, 4, 5]] @@ -740,7 +785,7 @@ def test_row_factories_with_dup_col_names(): def test_row_factories_with_reserved_word_col_names(): query = "select 1 as 'def'" - hndl = connect('mattdb', 'dev') + hndl = connect("mattdb", "dev") assert list(hndl.cursor().execute(query)) == [[1]] @@ -749,21 +794,21 @@ def test_row_factories_with_reserved_word_col_names(): hndl.cursor().execute(query) hndl.row_factory = dict_row_factory - assert list(hndl.cursor().execute(query)) == [{'def': 1}] + assert list(hndl.cursor().execute(query)) == [{"def": 1}] def test_reusing_handle_after_unicode_decode_error(): - hndl = connect('mattdb', 'dev') + hndl = connect("mattdb", "dev") cursor = hndl.cursor() with pytest.raises(DataError): cursor.execute("select cast(X'C3' as text)").fetchall() row = cursor.execute("select cast(X'C3A4' as text)").fetchone() - assert row == ['\xE4'] + assert row == ["\xe4"] def test_unicode_column_decode_exception(): query = "select cast(X'C3A4' as text) as a, cast(X'C3' as text) as b" - hndl = connect('mattdb', 'dev') + hndl = connect("mattdb", "dev") cursor = hndl.cursor() cursor.execute(query) @@ -779,7 +824,7 @@ def test_unicode_column_decode_exception(): def test_date_column_decode_exception(): query = "select cast('0000-01-01 UTC' as date) as date" - hndl = connect('mattdb', 'dev') + hndl = connect("mattdb", "dev") cursor = hndl.cursor() cursor.execute("SET TIMEZONE America/New_York") @@ -794,21 +839,23 @@ def test_date_column_decode_exception(): def test_unsupported_column_decode_exception(): query = "select now() - now() as delta" - hndl = connect('mattdb', 'dev') + hndl = connect("mattdb", "dev") cursor = hndl.cursor() cursor.execute(query) with pytest.raises(NotSupportedError) as exc_info: cursor.fetchall() - errmsg = ("Failed to decode CDB2_INTERVALDS column 0 ('delta'):" - " Unsupported column type") + errmsg = ( + "Failed to decode CDB2_INTERVALDS column 0 ('delta'):" + " Unsupported column type" + ) assert errmsg in str(exc_info.value) def test_datetimeus(): query = "select %(date)s + cast(30 as days)" - hndl = connect('mattdb', 'dev') + hndl = connect("mattdb", "dev") cursor = hndl.cursor() sent = DatetimeUs(2017, 8, 16, 19, 32, 2, 825022, tzinfo=pytz.UTC) @@ -819,7 +866,7 @@ def test_datetimeus(): def test_interface_error_reading_result_set_after_commits(): - hndl = connect('mattdb', 'dev') + hndl = connect("mattdb", "dev") cursor = hndl.cursor().execute("delete from simple where 1=1") assert cursor.description is None hndl.commit() @@ -827,7 +874,7 @@ def test_interface_error_reading_result_set_after_commits(): cursor.fetchall() assert "No result set exists" in str(exc_info.value) - hndl = connect('mattdb', 'dev', autocommit=True) + hndl = connect("mattdb", "dev", autocommit=True) cursor = hndl.cursor().execute("delete from simple where 1=1") assert cursor.description is None with pytest.raises(InterfaceError) as exc_info: @@ -835,21 +882,24 @@ def test_interface_error_reading_result_set_after_commits(): assert "No result set exists" in str(exc_info.value) -@pytest.mark.parametrize("statement, operation", [ - ("select 1", "select"), - (" \n\tselect 1", "select"), - (" /* */ select 1", "select"), - ("\n--\n\t\tselect 1", "select"), - ("insert/**/1", "insert"), - ("-- select 1\n-- Insert 2\nUpdate 3", "update"), - ("/*select 1*/\n/*Insert 2*/\nUpdate 3", "update"), - ("/* foo\nbar\nbaz\n-- INSERT*/select foo", "select"), - ("/* foo\nbar\nbaz\n-- INSERT*/--select\ninsert foo", "insert"), - ("-- /*\n*/ foo", None), - ("--", None), - ("*/", None), - ("", None), -]) +@pytest.mark.parametrize( + "statement, operation", + [ + ("select 1", "select"), + (" \n\tselect 1", "select"), + (" /* */ select 1", "select"), + ("\n--\n\t\tselect 1", "select"), + ("insert/**/1", "insert"), + ("-- select 1\n-- Insert 2\nUpdate 3", "update"), + ("/*select 1*/\n/*Insert 2*/\nUpdate 3", "update"), + ("/* foo\nbar\nbaz\n-- INSERT*/select foo", "select"), + ("/* foo\nbar\nbaz\n-- INSERT*/--select\ninsert foo", "insert"), + ("-- /*\n*/ foo", None), + ("--", None), + ("*/", None), + ("", None), + ], +) def test_finding_operation(statement, operation): assert _sql_operation(statement) == operation From 626a2187205c16902a41e1df5ddc8e74f4d4835d Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Wed, 10 Apr 2024 13:47:31 -0400 Subject: [PATCH 18/62] Use types from `collections.abc` for annotations PEP 585 allows the types in `collections.abc` to be used as generic types for the purposes of type annotations. Since we're using from __future__ import annotations we can use this feature even for Python 3.8, which predates PEP 585. Signed-off-by: Matt Wozniski --- comdb2/cdb2.py | 3 ++- comdb2/dbapi2.py | 3 ++- comdb2/factories.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/comdb2/cdb2.py b/comdb2/cdb2.py index 0666afb..d077bec 100644 --- a/comdb2/cdb2.py +++ b/comdb2/cdb2.py @@ -131,7 +131,8 @@ from __future__ import annotations import datetime -from typing import Any, Callable, Iterator, List, Mapping, Tuple, Union +from collections.abc import Callable, Iterator, Mapping +from typing import Any, List, Tuple, Union from ._cdb2_types import Error, Effects, DatetimeUs from ._ccdb2 import Handle as CHandle diff --git a/comdb2/dbapi2.py b/comdb2/dbapi2.py index 6741a24..c8622bd 100644 --- a/comdb2/dbapi2.py +++ b/comdb2/dbapi2.py @@ -250,7 +250,8 @@ from . import cdb2 from .cdb2 import Row, Value -from typing import Any, Callable, Iterator, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence +from typing import Any __all__ = [ "apilevel", diff --git a/comdb2/factories.py b/comdb2/factories.py index da41058..363111d 100644 --- a/comdb2/factories.py +++ b/comdb2/factories.py @@ -27,7 +27,8 @@ from collections import namedtuple from collections import Counter from .cdb2 import Value -from typing import Callable, NamedTuple +from collections.abc import Callable +from typing import NamedTuple def namedtuple_row_factory(col_names: list[str]) -> Callable[[list[Value]], NamedTuple]: From 59bb7064299985f83d35d6edd0b529e51b8e4186 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Wed, 10 Apr 2024 14:42:07 -0400 Subject: [PATCH 19/62] docs: Replace auto-generated NamedTuple docstrings Without this change, Sphinx complains that these attributes are documented twice, once by our Sphinx markup in the class docstring and once with a generated docstring for each property produced by `NamedTuple` itself for each property. Override the generated one with our description so that Sphinx finds only one docstring for each attribute, and so that `help()` shows the more useful one. Signed-off-by: Matt Wozniski --- comdb2/_cdb2_types.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/comdb2/_cdb2_types.py b/comdb2/_cdb2_types.py index 394a93b..9251bad 100644 --- a/comdb2/_cdb2_types.py +++ b/comdb2/_cdb2_types.py @@ -49,13 +49,6 @@ class Effects(NamedTuple): """Type used to represent the count of rows affected by a SQL query. An object of this type is returned by `Handle.get_effects`. - - Attributes: - num_affected (int): The total number of rows that were affected. - num_selected (int): The number of rows that were selected. - num_updated (int): The number of rows that were updated. - num_deleted (int): The number of rows that were deleted. - num_inserted (int): The number of rows that were inserted. """ num_affected: int @@ -65,6 +58,14 @@ class Effects(NamedTuple): num_inserted: int +# Override the auto-generated docstrings with more informative ones. +Effects.num_affected.__doc__ = "The total number of rows that were affected." +Effects.num_selected.__doc__ = "The number of rows that were selected." +Effects.num_updated.__doc__ = "The number of rows that were updated." +Effects.num_deleted.__doc__ = "The number of rows that were deleted." +Effects.num_inserted.__doc__ = "The number of rows that were inserted." + + class DatetimeUs(datetime.datetime): """Provides a distinct representation for Comdb2's DATETIMEUS type. From 42f37d03611a283231753d65e73d7e913bff8f4e Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Tue, 30 Apr 2024 14:42:27 -0400 Subject: [PATCH 20/62] Begin building as C++ rather than C99 This allows us to use the C++ standard library to more easily implement future changes, and reduces friction with some internal patches. Signed-off-by: Matt Wozniski --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 432e302..9a24fa5 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys import pkgconfig from setuptools import setup, Extension @@ -36,7 +35,7 @@ def make_static_extension(name, **kwargs): ccdb2 = make_static_extension( "comdb2._ccdb2", - extra_compile_args=["-std=c99"], + language="c++", libraries=["cdb2api"], sources=["comdb2/_ccdb2.pyx"], ) From 2a901beb1946a5141bb1b35183d3b3a98a782ead Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Wed, 27 Mar 2024 18:52:13 -0400 Subject: [PATCH 21/62] Let users request specific result set column types Expose a `column_types` keyword-only argument to `cdb2.Handle.execute` and `dbapi2.Cursor.execute` for specifying the types that the result set columns must have. The database will coerce returned values to the given type for the column. An exception will be raised if it can't, or if too few or too many types are given. Also, populate an `enum.IntEnum` with the acceptable column types. Previously we had this only as the members of a dict, but that doesn't play nicely with type annotations. Keep the dict for backwards compatibility, but use the enum to populate it. Signed-off-by: Matt Wozniski --- comdb2/_ccdb2.pyx | 11 +++-- comdb2/_cdb2_types.py | 46 ++++++++++++++++++++ comdb2/_cdb2api.pxd | 1 + comdb2/cdb2.py | 66 ++++++++++++++++------------- comdb2/dbapi2.py | 31 +++++++++++--- docs/cdb2.rst | 10 +++-- docs/dbapi2.rst | 5 +++ tests/test_cdb2.py | 41 ++++++++++++++++++ tests/test_dbapi2.py | 98 +++++++++++++++++++++++++++++++------------ 9 files changed, 241 insertions(+), 68 deletions(-) diff --git a/comdb2/_ccdb2.pyx b/comdb2/_ccdb2.pyx index 2b508fe..4b8e8fc 100644 --- a/comdb2/_ccdb2.pyx +++ b/comdb2/_ccdb2.pyx @@ -391,8 +391,8 @@ cdef class Handle(object): self.hndl = NULL _errchk(rc, self.hndl) - def execute(self, sql, parameters=None): - """execute(sql, params) -> Cursor over a SQL statement's result set""" + def execute(self, sql, parameters=None, int[::1] column_types=None): + """execute(sql, params, types) -> Cursor over a SQL statement's result set""" if not self.hndl: raise _closed_connection_error('execute') self.cursor = NULL sql = _string_as_bytes(sql) @@ -417,7 +417,12 @@ cdef class Handle(object): with nogil: while lib.cdb2_next_record(self.hndl) == lib.CDB2_OK: pass # consume any previous result set - rc = lib.cdb2_run_statement(self.hndl, c_sql) + if column_types is None: + rc = lib.cdb2_run_statement(self.hndl, c_sql) + else: + rc = lib.cdb2_run_statement_typed( + self.hndl, c_sql, len(column_types), &column_types[0] + ) _errchk(rc, self.hndl) finally: rc = lib.cdb2_clearbindings(self.hndl) diff --git a/comdb2/_cdb2_types.py b/comdb2/_cdb2_types.py index 9251bad..5f3f325 100644 --- a/comdb2/_cdb2_types.py +++ b/comdb2/_cdb2_types.py @@ -12,6 +12,7 @@ from __future__ import annotations import datetime +import enum from typing import NamedTuple __name__ = "comdb2.cdb2" @@ -145,3 +146,48 @@ def replace(self, *args, **kwargs): # this returns a DatetimeUs or a datetime. dt = super().replace(*args, **kwargs) return self.fromdatetime(dt) + + +class ColumnType(enum.IntEnum): + """This enum represents all known Comdb2 column types. + + Each value in the list returned by `Handle.column_types` will generally be + the value corresponding to one of the members of this enumeration, though + that's not always guaranteed because new types can be added to the Comdb2 + server at any time. + + A sequence of consisting of members of this enum can be passed as the + *column_types* parameter of `Handle.execute`. The database will coerce the + returned data to the given column types, or return an error if it cannot. + """ + + # The column types from cdb2_coltype in cdb2api.h, plus a few aliases. + INTEGER = 1 + REAL = 2 + CSTRING = 3 + BLOB = 4 + DATETIME = 6 + INTERVALYM = 7 + INTERVALDS = 8 + DATETIMEUS = 9 + INTERVALDSUS = 10 + INT = INTEGER + FLOAT = REAL + STRING = CSTRING + TEXT = CSTRING + BYTES = BLOB + + +class ConnectionFlags(enum.IntEnum): + """This enum represents connection flags. + + These values can be passed to the `Handle` constructor, either individually + or as a bitwise OR of multiple flags. + """ + + # The flags from cdb2_hndl_alloc_flags in cdb2api.h + READ_INTRANS_RESULTS = 2 + DIRECT_CPU = 4 + RANDOM = 8 + RANDOMROOM = 16 + ROOM = 32 diff --git a/comdb2/_cdb2api.pxd b/comdb2/_cdb2api.pxd index afae380..1eca538 100644 --- a/comdb2/_cdb2api.pxd +++ b/comdb2/_cdb2api.pxd @@ -68,6 +68,7 @@ cdef extern from "cdb2api.h" nogil: int cdb2_get_effects(cdb2_hndl_tp *hndl, cdb2_effects_tp *effects); int cdb2_close(cdb2_hndl_tp* hndl); int cdb2_run_statement(cdb2_hndl_tp *hndl, const char *sql); + int cdb2_run_statement_typed(cdb2_hndl_tp *hndl, const char *sql, int num_types, int *types); int cdb2_numcolumns(cdb2_hndl_tp* hndl); const char* cdb2_column_name(cdb2_hndl_tp* hndl, int col); int cdb2_column_type(cdb2_hndl_tp* hndl, int col); diff --git a/comdb2/cdb2.py b/comdb2/cdb2.py index d077bec..c3318f2 100644 --- a/comdb2/cdb2.py +++ b/comdb2/cdb2.py @@ -130,10 +130,11 @@ from __future__ import annotations +import array import datetime -from collections.abc import Callable, Iterator, Mapping +from collections.abc import Callable, Iterator, Mapping, Sequence from typing import Any, List, Tuple, Union -from ._cdb2_types import Error, Effects, DatetimeUs +from ._cdb2_types import Error, Effects, DatetimeUs, ColumnType, ConnectionFlags from ._ccdb2 import Handle as CHandle __all__ = [ @@ -144,6 +145,8 @@ "ERROR_CODE", "TYPE", "HANDLE_FLAGS", + "ColumnType", + "ConnectionFlags", ] Value = Union[ @@ -210,37 +213,18 @@ time. """ -# Pull comdb2 column types from cdb2api.h into our namespace -TYPE = { - "INTEGER": 1, - "REAL": 2, - "CSTRING": 3, - "BLOB": 4, - "DATETIME": 6, - "INTERVALYM": 7, - "INTERVALDS": 8, - "DATETIMEUS": 9, - "INTERVALDSUS": 10, -} +TYPE = {e.name: e.value for e in ColumnType} """This dict maps all known Comdb2 types to their enumeration value. -Each value in the list returned by `Handle.column_types` will generally be the -value corresponding to one of the keys in this dict, though that's not always -guaranteed because new types can be added to the Comdb2 server at any time. +It predates the `ColumnType` enum and is retained only for backwards +compatibility. The `ColumnType` enum should be preferred for new usage. """ -# Pull comdb2 handle flags from cdb2api.h into our namespace -HANDLE_FLAGS = { - "READ_INTRANS_RESULTS": 2, - "DIRECT_CPU": 4, - "RANDOM": 8, - "RANDOMROOM": 16, - "ROOM": 32, -} -"""This dict maps all known Comdb2 flags to their enumeration value. +HANDLE_FLAGS = {e.name: e.value for e in ConnectionFlags} +"""This dict maps connection flags to their enumeration value. -These values can be passed directly to `Handle`, though values not in this dict -can be passed as well (such as the bitwise OR of two different flags). +It predates the `ConnectionFlags` enum and is retained only for backwards +compatibility. The `ConnectionFlags` enum should be preferred for new usage. """ @@ -362,7 +346,11 @@ def row_factory( self._hndl.row_factory = value def execute( - self, sql: str | bytes, parameters: Mapping[str, Value] | None = None + self, + sql: str | bytes, + parameters: Mapping[str, Value] | None = None, + *, + column_types: Sequence[ColumnType] | None = None, ) -> Handle: """Execute a database operation (query or command). @@ -372,10 +360,21 @@ def execute( Placeholders for named parameters must be in Comdb2's native format, ``@param_name``. + If ``column_types`` is provided and non-empty, it must be a sequence of + members of the `ColumnType` enumeration. The database will coerce the + data in the Nth column of the result set to the Nth given column type. + An error will be raised if the number of elements in ``column_types`` + doesn't match the number of columns in the result set, or if one of the + elements is not a supported column type, or if coercion fails. If + ``column_types`` is empty or not provided, no coercion is performed. + Args: sql (str): The SQL string to execute. parameters (Mapping[str, Any]): An optional mapping from parameter names to the values to be bound for them. + column_types (Sequence[int]): An optional sequence of types (values + of the `ColumnType` enumeration) which the columns of the + result set will be coerced to. Returns: Handle: This method returns the `Handle` that it was called on, @@ -396,7 +395,14 @@ def execute( """ if parameters is None: parameters = {} - self._cursor = iter(self._hndl.execute(sql, parameters)) + + column_types_array: Sequence[int] | None = None + if column_types is not None: + column_types_array = array.array("i", column_types) + if not column_types_array: + column_types_array = None + + self._cursor = iter(self._hndl.execute(sql, parameters, column_types_array)) return self def __iter__(self) -> Iterator[Row]: diff --git a/comdb2/dbapi2.py b/comdb2/dbapi2.py index c8622bd..fd22678 100644 --- a/comdb2/dbapi2.py +++ b/comdb2/dbapi2.py @@ -249,7 +249,7 @@ import re from . import cdb2 -from .cdb2 import Row, Value +from .cdb2 import ColumnType, Row, Value from collections.abc import Callable, Iterator, Mapping, Sequence from typing import Any @@ -258,6 +258,7 @@ "threadsafety", "paramstyle", "connect", + "ColumnType", "Connection", "Cursor", "STRING", @@ -991,7 +992,11 @@ def callproc(self, procname: str, parameters: Sequence[Value]) -> Sequence[Value return parameters[:] def execute( - self, sql: str, parameters: Mapping[str, Value] | None = None + self, + sql: str, + parameters: Mapping[str, Value] | None = None, + *, + column_types: Sequence[ColumnType] | None = None, ) -> Cursor: """Execute a database operation (query or command). @@ -1005,10 +1010,26 @@ def execute( vulnerabilities, and is faster than dynamically building SQL strings. + If ``column_types`` is provided and non-empty, it must be a sequence of + members of the `ColumnType` enumeration. The database will coerce the + data in the Nth column of the result set to the Nth given column type. + An error will be raised if the number of elements in ``column_types`` + doesn't match the number of columns in the result set, or if one of the + elements is not a supported column type, or if coercion fails. If + ``column_types`` is empty or not provided, no coercion is performed. + + Note: + Databases APIs are not required to allow result set column types to + be specified explicitly. We allow this as a non-standard DB-API 2.0 + extension. + Args: sql (str): The SQL string to execute, as a Python format string. parameters (Mapping[str, Any]): An optional mapping from parameter names to the values to be bound for them. + column_types (Sequence[int]): An optional sequence of types (values + of the `ColumnType` enumeration) which the columns of the + result set will be coerced to. Returns: Cursor: As a nonstandard DB-API 2.0 extension, this method returns @@ -1037,7 +1058,7 @@ def execute( if errmsg: raise InterfaceError(errmsg) - self._execute(operation, sql, parameters) + self._execute(operation, sql, parameters, column_types=column_types) if self._rowcount == -1: self._load_description() # Optional DB API Extension: execute's return value is unspecified. We @@ -1064,7 +1085,7 @@ def executemany( for parameters in seq_of_parameters: self.execute(sql, parameters) - def _execute(self, operation, sql, parameters=None): + def _execute(self, operation, sql, parameters=None, *, column_types=None): self._rowcount = -1 if not self._conn._autocommit: @@ -1094,7 +1115,7 @@ def _execute(self, operation, sql, parameters=None): self._conn._in_transaction = False # txn ends, even on failure try: - self._hndl.execute(sql, parameters) + self._hndl.execute(sql, parameters, column_types=column_types) except cdb2.Error as e: _raise_wrapped_exception(e) diff --git a/docs/cdb2.rst b/docs/cdb2.rst index 83ff082..134a52a 100644 --- a/docs/cdb2.rst +++ b/docs/cdb2.rst @@ -46,8 +46,12 @@ Enumerations Several mappings are provided to expose enumerations from ``cdb2api.h`` to Python. - .. autodata:: TYPE +.. autoclass:: ColumnType - .. autodata:: HANDLE_FLAGS +.. autodata:: TYPE - .. autodata:: ERROR_CODE +.. autoclass:: ConnectionFlags + +.. autodata:: HANDLE_FLAGS + +.. autodata:: ERROR_CODE diff --git a/docs/dbapi2.rst b/docs/dbapi2.rst index 4030a90..f5d87e8 100644 --- a/docs/dbapi2.rst +++ b/docs/dbapi2.rst @@ -124,6 +124,11 @@ for consistency with the required DATETIME constructors documented above. after the Unix epoch, with microsecond precision :rtype: `DatetimeUs` +.. class:: ColumnType + + This is an alias for `comdb2.cdb2.ColumnType`, reexported from + ``comdb2.dbapi2`` for convenience. + .. _Exceptions: Exceptions diff --git a/tests/test_cdb2.py b/tests/test_cdb2.py index d9c0203..da8b736 100644 --- a/tests/test_cdb2.py +++ b/tests/test_cdb2.py @@ -9,6 +9,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import datetime from comdb2 import cdb2 from comdb2.factories import dict_row_factory @@ -301,3 +302,43 @@ def test_parameter_name_in_binding_errors_noexception(): assert exc.value.args[1] == ( "Can't map complex value (5+6j) for parameter 'param' to a Comdb2 type" ) + + +def test_specifying_column_types(): + # GIVEN + hndl = cdb2.Handle("mattdb", "dev") + column_types = [ + cdb2.ColumnType.INT, + cdb2.ColumnType.REAL, + cdb2.ColumnType.TEXT, + cdb2.ColumnType.BLOB, + cdb2.ColumnType.DATETIME, + cdb2.ColumnType.DATETIMEUS, + ] + sql = "select 0, 0, 0, 0, 0, 0" + + # WHEN + (row,) = hndl.execute(sql, column_types=column_types) + + # THEN + assert row is not None + assert type(row[0]) is int + assert type(row[1]) is float + assert type(row[2]) is str + assert type(row[3]) is bytes + assert type(row[4]) is datetime.datetime + assert type(row[5]) is cdb2.DatetimeUs + + +def test_providing_empty_column_types_array(): + # GIVEN + hndl = cdb2.Handle("mattdb", "dev") + column_types = [] + sql = "select 0" + + # WHEN + (row,) = hndl.execute(sql, column_types=column_types) + + # THEN + assert row is not None + assert type(row[0]) is int diff --git a/tests/test_dbapi2.py b/tests/test_dbapi2.py index db51f53..3dbd175 100644 --- a/tests/test_dbapi2.py +++ b/tests/test_dbapi2.py @@ -10,37 +10,39 @@ # limitations under the License. -from comdb2.dbapi2 import _sql_operation -from comdb2.dbapi2 import NUMBER -from comdb2.dbapi2 import BINARY -from comdb2.dbapi2 import STRING -from comdb2.dbapi2 import DATETIME -from comdb2.dbapi2 import connect -from comdb2.dbapi2 import Binary -from comdb2.dbapi2 import Datetime -from comdb2.dbapi2 import DatetimeUs -from comdb2.dbapi2 import Timestamp -from comdb2.dbapi2 import TimestampUs -from comdb2.dbapi2 import DataError -from comdb2.dbapi2 import OperationalError -from comdb2.dbapi2 import IntegrityError -from comdb2.dbapi2 import ForeignKeyConstraintError -from comdb2.dbapi2 import NonNullConstraintError -from comdb2.dbapi2 import UniqueKeyConstraintError -from comdb2.dbapi2 import InterfaceError -from comdb2.dbapi2 import NotSupportedError -from comdb2.dbapi2 import ProgrammingError -from comdb2 import cdb2 -from comdb2.factories import dict_row_factory -from comdb2.factories import namedtuple_row_factory -import pytest import datetime -import pytz import re from functools import partial - from unittest.mock import patch +import pytest +import pytz + +from comdb2 import cdb2 +from comdb2.dbapi2 import ( + BINARY, + DATETIME, + NUMBER, + STRING, + Binary, + DataError, + Datetime, + DatetimeUs, + ForeignKeyConstraintError, + IntegrityError, + InterfaceError, + NonNullConstraintError, + NotSupportedError, + OperationalError, + ProgrammingError, + Timestamp, + TimestampUs, + UniqueKeyConstraintError, + _sql_operation, + connect, +) +from comdb2.factories import dict_row_factory, namedtuple_row_factory + COLUMN_LIST = ( "short_col u_short_col int_col u_int_col longlong_col" " float_col double_col byte_col byte_array_col" @@ -663,7 +665,7 @@ def test_exceptions_containing_unicode_error_messages(): raise -def throw_on(expected_stmt, stmt, parameters=None): +def throw_on(expected_stmt, stmt, parameters=None, column_types=None): if stmt == expected_stmt: raise cdb2.Error(42, "Not supported error") @@ -965,3 +967,45 @@ def test_parameter_binding_invalid_arrays(values, exc_msg): # WHEN/THEN with pytest.raises(DataError, match=re.escape(exc_msg)): cursor.execute("select * from carray(%(values)s)", dict(values=values)) + + +def test_specifying_column_types(): + # GIVEN + conn = connect("mattdb", "dev") + cursor = conn.cursor() + column_types = [ + cdb2.ColumnType.INT, + cdb2.ColumnType.REAL, + cdb2.ColumnType.TEXT, + cdb2.ColumnType.BLOB, + cdb2.ColumnType.DATETIME, + cdb2.ColumnType.DATETIMEUS, + ] + sql = "select 0, 0, 0, 0, 0, 0" + + # WHEN + cursor.execute(sql, column_types=column_types) + row = cursor.fetchone() + + # THEN + assert row is not None + assert type(row[0]) is int + assert type(row[1]) is float + assert type(row[2]) is str + assert type(row[3]) is bytes + assert type(row[4]) is datetime.datetime + assert type(row[5]) is cdb2.DatetimeUs + + +def test_providing_empty_column_types_array(): + # GIVEN + hndl = cdb2.Handle("mattdb", "dev") + column_types = [] + sql = "select 0" + + # WHEN + (row,) = hndl.execute(sql, column_types=column_types) + + # THEN + assert row is not None + assert type(row[0]) is int From d9cf62f41ddb9ee718ca5a3a44d722950aca1130 Mon Sep 17 00:00:00 2001 From: Sarah Monod Date: Mon, 1 Jul 2024 08:47:55 -0400 Subject: [PATCH 22/62] ci: Start building and testing for Python 3.13 Now that 3.13 beta is out, we should start this Signed-off-by: Sarah Monod --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0430d55..773e713 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -114,6 +114,7 @@ jobs: strategy: matrix: python-version: + - '3.13-dev' - '3.12' - '3.11' - '3.10' From 0c32881a2e5db04b4d58071cd0cf7a1c239d6e33 Mon Sep 17 00:00:00 2001 From: Sarah Monod Date: Mon, 1 Jul 2024 17:35:14 -0400 Subject: [PATCH 23/62] ci: Fix python matrix access MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Up until now, we were always testing with `python3.10`, no matter which version was setup… which made all the tests green, but also only tested one version instead of all versions as was the intention. Signed-off-by: Sarah Monod --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 773e713..5f1a6f4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -180,10 +180,10 @@ jobs: echo "Creating $table_name from $table_file" /opt/bb/bin/cdb2sql "$COMDB2_DBNAME" local "create table $table_name { $(cat $table_file) }" done - - name: Set up Python ${{matrix.python_version}} + - name: Set up Python ${{matrix.python-version}} uses: actions/setup-python@v4 # note that this step overwrites the PKG_CONFIG_PATH variable with: - python-version: "${{matrix.python_version}}" + python-version: "${{matrix.python-version}}" - name: Install python-comdb2 from the sdist run: ' sudo apt-get install -qy pkg-config && From aae339549641a29bcd5955106af29f964c4e0d48 Mon Sep 17 00:00:00 2001 From: Saharsh Rathi Date: Sat, 3 Aug 2024 10:14:56 +0100 Subject: [PATCH 24/62] Declare `fetchmany` and `fetchall` to return lists Previously their annotations used the less specific `Sequence` type, despite the docstring saying that lists are returned. This caused type checkers to reject concatenating two result sets with `+`. Signed-off-by: Saharsh Rathi --- comdb2/dbapi2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/comdb2/dbapi2.py b/comdb2/dbapi2.py index fd22678..acb49f8 100644 --- a/comdb2/dbapi2.py +++ b/comdb2/dbapi2.py @@ -251,7 +251,7 @@ from . import cdb2 from .cdb2 import ColumnType, Row, Value from collections.abc import Callable, Iterator, Mapping, Sequence -from typing import Any +from typing import Any, List __all__ = [ "apilevel", @@ -1167,7 +1167,7 @@ def fetchone(self) -> Row | None: except StopIteration: return None - def fetchmany(self, n: int | None = None) -> Sequence[Row]: + def fetchmany(self, n: int | None = None) -> List[Row]: """Fetch the next set of rows of the current result set. Args: @@ -1187,7 +1187,7 @@ def fetchmany(self, n: int | None = None) -> Sequence[Row]: n = self._arraysize return [x for x in itertools.islice(self, 0, n)] - def fetchall(self) -> Sequence[Row]: + def fetchall(self) -> List[Row]: """Fetch all remaining rows of the current result set. Returns: From ea2564401620170aabbda986dcd698ce6863f6e1 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Wed, 4 Sep 2024 14:22:15 -0400 Subject: [PATCH 25/62] Prepare for 1.7.0 release Signed-off-by: Matt Wozniski --- comdb2/_about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comdb2/_about.py b/comdb2/_about.py index 73f175b..eb2199c 100644 --- a/comdb2/_about.py +++ b/comdb2/_about.py @@ -1,2 +1,2 @@ __all__ = ["__version__"] -__version__ = "1.6.1" +__version__ = "1.7.0" From 92f90f1eb7e5e7eaa20a2bade566004a10eea434 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Wed, 4 Sep 2024 19:49:47 -0400 Subject: [PATCH 26/62] docs: Document array binding When we added this feature, we didn't update the documentation to describe it. Signed-off-by: Matt Wozniski --- docs/tips.rst | 26 ++++++++++++++++++++++++-- docs/types.rst | 23 +++++++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/docs/tips.rst b/docs/tips.rst index f4e91ff..ba12033 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -56,6 +56,13 @@ Best Practices, Tips, and Tricks Because of this: https://xkcd.com/327/ + When using comdb2 R8, you can even bind a `list` or `tuple` object like so:: + + c.execute( + "SELECT %(needle)s IN CARRAY(%(haystack)s)", + {'needle': 5, 'haystack': [1, 2, 3, 4, 5]} + ) + .. note:: The two modules provided by this package use different syntax for SQL placeholders. See `.dbapi2.Cursor.execute` and `.cdb2.Handle.execute` @@ -79,8 +86,23 @@ Best Practices, Tips, and Tricks `.cdb2.Handle` (any isolation level higher than ``READ COMMITTED`` would obviously work as well). -#. The underlying API doesn't currently allow binding lists. The following snippet - will be useful for a ``$var in $list`` query with a dynamically generated list:: +#. When using comdb2 R8, the library supports binding of (non-empty) `list` or + `tuple` objects with the help of the ``CARRAY`` function, as long as all + elements of the sequence are of the same type:: + + from comdb2.dbapi2 import connect + + def search_in_list(needle, haystack): + params = {'needle': needle, 'haystack': haystack} + sql = "select %(needle)s in CARRAY(%(haystack)s)" + print(connect('mattdb').cursor().execute(sql, params).fetchall()) + + haystack = [1, 2, 3, 4, 5] + search_in_list(0, haystack) + search_in_list(5, haystack) + + If you still need to use R7, you would want to generate the query + dynamically, for example:: from comdb2.dbapi2 import connect diff --git a/docs/types.rst b/docs/types.rst index 79bfaf5..9514526 100644 --- a/docs/types.rst +++ b/docs/types.rst @@ -18,6 +18,7 @@ blob `bytes` text `str` datetime `datetime.datetime` datetimeus `~.cdb2.DatetimeUs` +carray `list` or `tuple` of exactly one of the types above (R8 only) intervalym not supported intervalds not supported intervaldsus not supported @@ -111,3 +112,25 @@ string columns. That currently makes it impossible to map them to Python's `decimal.Decimal` type as we would like to. If ``libcdb2api`` is ever changed to properly distinguish between DECIMAL and TEXT columns this package will be enhanced to properly expose DECIMAL columns. + +C Array Types +============= + +.. note:: + This only works with comdb2 R8. If you need to connect to a database that is + still running comdb2 R7, the queries using this feature will fail. + +Often in SQL queries, it is useful to bind an array of elements of the same +type, for example when using the ``IN`` operator. In this case, you can use a +(non-empty) `list` or `tuple`, as long as all elements of the sequence are of +the same type (and one of the above Python types). Note that nested sequences +are not allowed. + +Note that when binding an element of that type, you want to use the ``CARRAY`` +function, like so:: + + params = {'arr': [1, 2, 3, 4]} + # dbapi2 + c.execute("select 2 in CARRAY(%(arr)s)", params) + # cdb2 + h.execute("select 2 in CARRAY(@arr)", params) From 3b5a2c0bf1b54053c990cfce7b39a5830475e001 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Wed, 4 Sep 2024 21:15:36 -0400 Subject: [PATCH 27/62] Update `upload-artifact` and `download-artifact` The latest version of each is v4, and v3 is deprecated and scheduled to be decommissioned. Signed-off-by: Matt Wozniski --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5f1a6f4..f752b61 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -61,7 +61,7 @@ jobs: - name: Archive bloomberg-comdb2 repo with build artifacts run: 'tar czvf bloomberg-comdb2.tar.gz bloomberg-comdb2/' - name: Upload bloomberg-comdb2 repo with build artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: bloomberg-comdb2 path: ./bloomberg-comdb2.tar.gz @@ -72,7 +72,7 @@ jobs: needs: [build_bloomberg_comdb2] steps: - name: Download bloomberg-comdb2 with build artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: bloomberg-comdb2 path: . @@ -102,7 +102,7 @@ jobs: pipx run build --sdist ' - name: Upload sdist - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: python-comdb2-sdist path: dist/*.tar.gz @@ -123,7 +123,7 @@ jobs: - '3.7' steps: - name: Download comdb2 repo with build artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: bloomberg-comdb2 path: . @@ -162,7 +162,7 @@ jobs: done ' - name: Download python-comdb2 sdist - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: python-comdb2-sdist path: dist From 5a1ff82edd5f336a91203c1d704ff705367c6ae9 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Wed, 4 Sep 2024 21:48:22 -0400 Subject: [PATCH 28/62] Enable dependabot for upgrading GitHub actions Signed-off-by: Matt Wozniski --- .github/dependabot.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..1230149 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" From 1b1e593c3fdfb15c0f6f1432651b7cc0065eb524 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Sep 2024 12:40:13 +0000 Subject: [PATCH 29/62] Bump actions/setup-python from 4 to 5 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f752b61..0c45c4f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -92,7 +92,7 @@ jobs: ' - uses: actions/checkout@v4 - name: 'Set up Python 3.7' - uses: actions/setup-python@v4 # note that this step overwrites the PKG_CONFIG_PATH variable + uses: actions/setup-python@v5 # note that this step overwrites the PKG_CONFIG_PATH variable with: python-version: '3.7' # the lowest version that we support in CI - name: Build sdist @@ -181,7 +181,7 @@ jobs: /opt/bb/bin/cdb2sql "$COMDB2_DBNAME" local "create table $table_name { $(cat $table_file) }" done - name: Set up Python ${{matrix.python-version}} - uses: actions/setup-python@v4 # note that this step overwrites the PKG_CONFIG_PATH variable + uses: actions/setup-python@v5 # note that this step overwrites the PKG_CONFIG_PATH variable with: python-version: "${{matrix.python-version}}" - name: Install python-comdb2 from the sdist From f1a68c58c119c3b4e2ae3c0e8930505959e882f8 Mon Sep 17 00:00:00 2001 From: Sarah Monod Date: Thu, 5 Sep 2024 08:36:07 -0400 Subject: [PATCH 30/62] Handle C++ exceptions thrown within `libcdb2api` While `libcdb2api` itself is implemented in C, the version used internally at Bloomberg can optionally call into some plugins that are implemented in C++, which could raise exceptions. Ensure that `python-comdb2` catches any such exceptions, both for the benefit of Bloomberg's internal Comdb2 users and to guard against the possibility that C++ code is added to `libcdb2api` itself in the future. Signed-off-by: Sarah Monod --- comdb2/_cdb2api.pxd | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/comdb2/_cdb2api.pxd b/comdb2/_cdb2api.pxd index 1eca538..e061680 100644 --- a/comdb2/_cdb2api.pxd +++ b/comdb2/_cdb2api.pxd @@ -63,19 +63,19 @@ cdef extern from "cdb2api.h" nogil: ctypedef struct cdb2_hndl_tp: pass - int cdb2_open(cdb2_hndl_tp **hndl, const char *dbname, const char *type, int flags); - int cdb2_next_record(cdb2_hndl_tp *hndl); - int cdb2_get_effects(cdb2_hndl_tp *hndl, cdb2_effects_tp *effects); - int cdb2_close(cdb2_hndl_tp* hndl); - int cdb2_run_statement(cdb2_hndl_tp *hndl, const char *sql); - int cdb2_run_statement_typed(cdb2_hndl_tp *hndl, const char *sql, int num_types, int *types); - int cdb2_numcolumns(cdb2_hndl_tp* hndl); - const char* cdb2_column_name(cdb2_hndl_tp* hndl, int col); - int cdb2_column_type(cdb2_hndl_tp* hndl, int col); - int cdb2_column_size(cdb2_hndl_tp* hndl, int col); - void* cdb2_column_value(cdb2_hndl_tp* hndl, int col); - const char* cdb2_errstr(cdb2_hndl_tp* hndl); - int cdb2_bind_param(cdb2_hndl_tp *hndl, const char *name, int type, const void *varaddr, int length); - int cdb2_bind_array(cdb2_hndl_tp *hndl, const char *name, cdb2_coltype, const void *varaddr, size_t count, size_t typelen); - int cdb2_clearbindings(cdb2_hndl_tp *hndl); - int cdb2_clear_ack(cdb2_hndl_tp *hndl); + int cdb2_open(cdb2_hndl_tp **hndl, const char *dbname, const char *type, int flags) except + + int cdb2_next_record(cdb2_hndl_tp *hndl) except + + int cdb2_get_effects(cdb2_hndl_tp *hndl, cdb2_effects_tp *effects) except + + int cdb2_close(cdb2_hndl_tp* hndl) except + + int cdb2_run_statement(cdb2_hndl_tp *hndl, const char *sql) except + + int cdb2_run_statement_typed(cdb2_hndl_tp *hndl, const char *sql, int num_types, int *types) except + + int cdb2_numcolumns(cdb2_hndl_tp* hndl) except + + const char* cdb2_column_name(cdb2_hndl_tp* hndl, int col) except + + int cdb2_column_type(cdb2_hndl_tp* hndl, int col) except + + int cdb2_column_size(cdb2_hndl_tp* hndl, int col) except + + void* cdb2_column_value(cdb2_hndl_tp* hndl, int col) except + + const char* cdb2_errstr(cdb2_hndl_tp* hndl) except + + int cdb2_bind_param(cdb2_hndl_tp *hndl, const char *name, int type, const void *varaddr, int length) except + + int cdb2_bind_array(cdb2_hndl_tp *hndl, const char *name, cdb2_coltype, const void *varaddr, size_t count, size_t typelen) except + + int cdb2_clearbindings(cdb2_hndl_tp *hndl) except + + int cdb2_clear_ack(cdb2_hndl_tp *hndl) except + From a4c0aa92db3576a11bae4f92d4b53444b791fbcc Mon Sep 17 00:00:00 2001 From: Sarah Monod Date: Thu, 5 Sep 2024 08:38:15 -0400 Subject: [PATCH 31/62] Prepare for 1.7.1 release Signed-off-by: Sarah Monod --- comdb2/_about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comdb2/_about.py b/comdb2/_about.py index eb2199c..9b5c503 100644 --- a/comdb2/_about.py +++ b/comdb2/_about.py @@ -1,2 +1,2 @@ __all__ = ["__version__"] -__version__ = "1.7.0" +__version__ = "1.7.1" From ef406ec665704e7e6c9d1fe2a827e9e13873d8ca Mon Sep 17 00:00:00 2001 From: Jonathan Surany Date: Mon, 23 Sep 2024 10:14:57 -0400 Subject: [PATCH 32/62] Export types for column and parameter values Update `comdb2.cdb2` and `comdb2.dbapi2` to export 2 `typing.Union` type aliases: - `Value`: the types that values representing result set columns may have - `ParameterValue`: the types that bound parameters of a SQL statement may have Signed-off-by: Jonathan Surany --- comdb2/cdb2.py | 13 ++++++++++++- comdb2/dbapi2.py | 10 ++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/comdb2/cdb2.py b/comdb2/cdb2.py index c3318f2..01f8c7f 100644 --- a/comdb2/cdb2.py +++ b/comdb2/cdb2.py @@ -147,6 +147,8 @@ "HANDLE_FLAGS", "ColumnType", "ConnectionFlags", + "Value", + "ParameterValue", ] Value = Union[ @@ -157,6 +159,15 @@ str, datetime.datetime, DatetimeUs, +] +ParameterValue = Union[ + None, + int, + float, + bytes, + str, + datetime.datetime, + DatetimeUs, List[int], List[float], List[bytes], @@ -348,7 +359,7 @@ def row_factory( def execute( self, sql: str | bytes, - parameters: Mapping[str, Value] | None = None, + parameters: Mapping[str, ParameterValue] | None = None, *, column_types: Sequence[ColumnType] | None = None, ) -> Handle: diff --git a/comdb2/dbapi2.py b/comdb2/dbapi2.py index acb49f8..581b685 100644 --- a/comdb2/dbapi2.py +++ b/comdb2/dbapi2.py @@ -249,7 +249,7 @@ import re from . import cdb2 -from .cdb2 import ColumnType, Row, Value +from .cdb2 import ColumnType, Row, Value, ParameterValue from collections.abc import Callable, Iterator, Mapping, Sequence from typing import Any, List @@ -287,6 +287,8 @@ "UniqueKeyConstraintError", "ForeignKeyConstraintError", "NonNullConstraintError", + "Value", + "ParameterValue", ] apilevel = "2.0" @@ -958,7 +960,7 @@ def close(self) -> None: self._description = None self._closed = True - def callproc(self, procname: str, parameters: Sequence[Value]) -> Sequence[Value]: + def callproc(self, procname: str, parameters: Sequence[ParameterValue]) -> Sequence[ParameterValue]: """Call a stored procedure with the given name. The ``parameters`` sequence must contain one entry for each argument @@ -994,7 +996,7 @@ def callproc(self, procname: str, parameters: Sequence[Value]) -> Sequence[Value def execute( self, sql: str, - parameters: Mapping[str, Value] | None = None, + parameters: Mapping[str, ParameterValue] | None = None, *, column_types: Sequence[ColumnType] | None = None, ) -> Cursor: @@ -1066,7 +1068,7 @@ def execute( return self def executemany( - self, sql: str, seq_of_parameters: Sequence[Mapping[str, Value]] + self, sql: str, seq_of_parameters: Sequence[Mapping[str, ParameterValue]] ) -> None: """Execute the same SQL statement repeatedly with different parameters. From 518dd75d3da2334462cb0e077e9683407163bb2b Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Tue, 1 Oct 2024 19:12:14 -0400 Subject: [PATCH 33/62] Prepare for 1.7.2 release Signed-off-by: Matt Wozniski --- comdb2/_about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comdb2/_about.py b/comdb2/_about.py index 9b5c503..8763b29 100644 --- a/comdb2/_about.py +++ b/comdb2/_about.py @@ -1,2 +1,2 @@ __all__ = ["__version__"] -__version__ = "1.7.1" +__version__ = "1.7.2" From cbb32dc199aa6f3ba68571ce946a516e40974345 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Tue, 14 Jan 2025 18:53:29 -0500 Subject: [PATCH 34/62] ci: Share built Comdb2 across jobs Previously we were sharing the Comdb2 build directory, but this didn't work entirely as intended, as the `make install` step could detect that some files needed to be rebuilt. Instead, run the `make install` once, and copy around the installed files. Signed-off-by: Matt Wozniski --- .github/workflows/build.yml | 37 ++++++++++++------------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0c45c4f..9eb9957 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -55,12 +55,13 @@ jobs: mkdir bloomberg-comdb2/build && cd bloomberg-comdb2/build && cmake .. && - make + make && + sudo make install ) ' - name: Archive bloomberg-comdb2 repo with build artifacts - run: 'tar czvf bloomberg-comdb2.tar.gz bloomberg-comdb2/' - - name: Upload bloomberg-comdb2 repo with build artifacts + run: 'tar -C /opt -czvf bloomberg-comdb2.tar.gz bb' + - name: Upload built bloomberg-comdb2 uses: actions/upload-artifact@v4 with: name: bloomberg-comdb2 @@ -78,17 +79,10 @@ jobs: path: . - name: Install bloomberg-comdb2 run: ' - sudo apt-get install -qy - libevent-dev - liblz4-dev - libprotobuf-c-dev - libsqlite3-dev - libssl-dev - libunwind-dev - zlib1g-dev && - tar xvf bloomberg-comdb2.tar.gz && - (cd bloomberg-comdb2/build && sudo make install) && - sudo rm -rf bloomberg-comdb2/ bloomberg-comdb2.tar.gz + sudo apt-get install -qy libprotobuf-c-dev libssl-dev libunwind-dev && + sudo tar -C /opt -xzf bloomberg-comdb2.tar.gz && + find /opt/bb && + rm bloomberg-comdb2.tar.gz ' - uses: actions/checkout@v4 - name: 'Set up Python 3.7' @@ -129,17 +123,10 @@ jobs: path: . - name: Install bloomberg-comdb2 run: ' - sudo apt-get install -qy - libevent-dev - liblz4-dev - libprotobuf-c-dev - libsqlite3-dev - libssl-dev - libunwind-dev - zlib1g-dev && - tar xvf bloomberg-comdb2.tar.gz && - (cd bloomberg-comdb2/build && sudo make install) && - sudo rm -rf bloomberg-comdb2/ bloomberg-comdb2.tar.gz + sudo apt-get install -qy libprotobuf-c-dev libssl-dev libunwind-dev && + sudo tar -C /opt -xzf bloomberg-comdb2.tar.gz && + find /opt/bb && + rm bloomberg-comdb2.tar.gz ' - name: Start local comdb2 instance run: ' From 637c004424046c0fcf2a941fe7f1fe567cfad6ab Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Tue, 14 Jan 2025 19:43:56 -0500 Subject: [PATCH 35/62] Drop support for Python 3.7 Signed-off-by: Matt Wozniski --- .github/workflows/build.yml | 5 ++--- setup.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9eb9957..7c52b38 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -85,10 +85,10 @@ jobs: rm bloomberg-comdb2.tar.gz ' - uses: actions/checkout@v4 - - name: 'Set up Python 3.7' + - name: 'Set up Python 3.8' uses: actions/setup-python@v5 # note that this step overwrites the PKG_CONFIG_PATH variable with: - python-version: '3.7' # the lowest version that we support in CI + python-version: '3.8' # the lowest version that we support in CI - name: Build sdist run: ' sudo apt-get install -qy pkg-config && @@ -114,7 +114,6 @@ jobs: - '3.10' - '3.9' - '3.8' - - '3.7' steps: - name: Download comdb2 repo with build artifacts uses: actions/download-artifact@v4 diff --git a/setup.py b/setup.py index 9a24fa5..fe7def5 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ def make_static_extension(name, **kwargs): packages=["comdb2"], install_requires=["pytz"], extras_require={"tests": ["python-dateutil>=2.6.0", "pytest"]}, - python_requires=">=3.7", + python_requires=">=3.8", ext_modules=[ccdb2], package_data={"comdb2": ["py.typed", "*.pyi"]}, ) From d36da372868e84f9102a74d5988727a66c1b5ae7 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Wed, 15 Jan 2025 16:16:59 -0500 Subject: [PATCH 36/62] ci: Drop "-dev" from requested Python 3.13 version Now that there has been a 3.13 final release, "3.13" and "3.13-dev" are equivalent. Signed-off-by: Matt Wozniski --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7c52b38..d3de70e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -108,7 +108,7 @@ jobs: strategy: matrix: python-version: - - '3.13-dev' + - '3.13' - '3.12' - '3.11' - '3.10' From f87217b01a7678625c4dd769e9824c8fc91e3e40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Tue, 14 Jan 2025 15:34:42 -0800 Subject: [PATCH 37/62] Add `as_datetime` method to `DatetimeUs` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This method can be used to explicitly convert a `DatetimeUs` instance to a `datetime` instance that can be used somewhere where instances of the subclass are not accepted (such as pydantic). Signed-off-by: Bernát Gábor --- comdb2/_cdb2_types.py | 14 ++++++++++++++ tests/test_cdb2_datetimeus.py | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/comdb2/_cdb2_types.py b/comdb2/_cdb2_types.py index 5f3f325..8b60d18 100644 --- a/comdb2/_cdb2_types.py +++ b/comdb2/_cdb2_types.py @@ -147,6 +147,20 @@ def replace(self, *args, **kwargs): dt = super().replace(*args, **kwargs) return self.fromdatetime(dt) + def as_datetime(self) -> datetime.datetime: + """Return a `datetime` representation.""" + return datetime.datetime( + year=self.year, + day=self.day, + month=self.month, + hour=self.hour, + minute=self.minute, + second=self.second, + microsecond=self.microsecond, + tzinfo=self.tzinfo, + fold=self.fold, + ) + class ColumnType(enum.IntEnum): """This enum represents all known Comdb2 column types. diff --git a/tests/test_cdb2_datetimeus.py b/tests/test_cdb2_datetimeus.py index 3589722..334ea56 100644 --- a/tests/test_cdb2_datetimeus.py +++ b/tests/test_cdb2_datetimeus.py @@ -91,6 +91,20 @@ def test_datetimeus_astimezone(): assert type(dtu.astimezone(pytz.utc)) == cdb2.DatetimeUs +def test_datetimeus_as_datetime_naive(): + dtu = cdb2.DatetimeUs.fromtimestamp(time.time()) + dt = dtu.as_datetime() + assert type(dt) == datetime.datetime + assert dt == dtu + + +def test_datetimeus_as_datetime_with_tz(): + dtu = cdb2.DatetimeUs.fromtimestamp(time.time(), tz=pytz.UTC) + dt = dtu.as_datetime() + assert type(dt) == datetime.datetime + assert dt == dtu + + def test_datetimeus_type_stickiness(): def check(obj): assert isinstance(obj, cdb2.DatetimeUs) From c4bd8c82ae64f45bf2bb548702e2dbf33341e497 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Fri, 28 Feb 2025 17:52:04 -0500 Subject: [PATCH 38/62] docs: Remove an unused setting from docs/conf.py This is now triggering warnings saying that: WARNING: The config value `viewcode_import' has type `bool'; expected `NoneType'. The comment says that we were trying to prevent the `viewcode` extension from generating HTML for the standard library `datetime` module, but even with this setting removed it doesn't do that. This may just be a holdover from some long ago version of Sphinx, but it no longer seems to be necessary or useful. Signed-off-by: Matt Wozniski --- docs/conf.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 0c57c46..ed54495 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -264,9 +264,6 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {"python": ("https://docs.python.org/3", None)} -# Don't highlight the code for the Python standard library datetime module. -viewcode_import = False - # Don't automatically infer types from type annotations. autodoc_typehints = "none" From 5c2e155a4dff0d43f000335b25208ee29dab04be Mon Sep 17 00:00:00 2001 From: Salil Chandra Date: Wed, 18 Dec 2024 14:51:27 -0500 Subject: [PATCH 39/62] Add support for binding parameters positionally Add support for `?` placeholders that are bound sequentially to elements of a sequence of parameters. Signed-off-by: Salil Chandra Signed-off-by: Matt Wozniski --- comdb2/_ccdb2.pyx | 36 +++++++++--- comdb2/_cdb2api.pxd | 2 + comdb2/cdb2.py | 68 ++++++++++++++++------ comdb2/dbapi2.py | 132 ++++++++++++++++++++++++++++++++++--------- docs/tips.rst | 23 ++++++-- tests/test_cdb2.py | 20 ++++++- tests/test_dbapi2.py | 116 ++++++++++++++++++++++++++++++++++--- 7 files changed, 329 insertions(+), 68 deletions(-) diff --git a/comdb2/_ccdb2.pyx b/comdb2/_ccdb2.pyx index 4b8e8fc..43b0bae 100644 --- a/comdb2/_ccdb2.pyx +++ b/comdb2/_ccdb2.pyx @@ -223,7 +223,7 @@ cdef class _ParameterValue(object): exc_desc = _describe_exception(exc) if exc is not None: - errmsg = "Can't bind %s value %r for parameter '%s': %s" % ( + errmsg = "Can't bind %s value %r for parameter %r: %s" % ( type(obj).__name__, obj, param_name, @@ -231,7 +231,7 @@ cdef class _ParameterValue(object): ) raise Error(lib.CDB2ERR_CONV_FAIL, errmsg) from exc else: - errmsg = "Can't map %s value %r for parameter '%s' to a Comdb2 type" % ( + errmsg = "Can't map %s value %r for parameter %r to a Comdb2 type" % ( type(obj).__name__, obj, param_name, @@ -401,17 +401,35 @@ cdef class Handle(object): param_guards = [] try: if parameters is not None: - for key, val in parameters.items(): - ckey = _string_as_bytes(key) + if hasattr(parameters, "items"): + items = parameters.items() + bind_by_index = False + else: + items = enumerate(parameters, 1) + bind_by_index = True + + for key, val in items: + ckey = key if bind_by_index else _string_as_bytes(key) cval = _ParameterValue(val, key) param_guards.append(ckey) param_guards.append(cval) - if cval.list_size == -1: - rc = lib.cdb2_bind_param(self.hndl, ckey, - cval.type, cval.data, cval.size) + bind_array = (cval.list_size != -1) + if bind_array: + if bind_by_index: + rc = lib.cdb2_bind_array_index(self.hndl, ckey, + cval.type, cval.data, + cval.list_size, cval.size) + else: + rc = lib.cdb2_bind_array(self.hndl, ckey, + cval.type, cval.data, + cval.list_size, cval.size) else: - # Bind Array if cval is an array - rc = lib.cdb2_bind_array(self.hndl, ckey, cval.type, cval.data, cval.list_size, cval.size) + if bind_by_index: + rc = lib.cdb2_bind_index(self.hndl, ckey, + cval.type, cval.data, cval.size) + else: + rc = lib.cdb2_bind_param(self.hndl, ckey, + cval.type, cval.data, cval.size) _errchk(rc, self.hndl) with nogil: diff --git a/comdb2/_cdb2api.pxd b/comdb2/_cdb2api.pxd index e061680..2dfe554 100644 --- a/comdb2/_cdb2api.pxd +++ b/comdb2/_cdb2api.pxd @@ -76,6 +76,8 @@ cdef extern from "cdb2api.h" nogil: void* cdb2_column_value(cdb2_hndl_tp* hndl, int col) except + const char* cdb2_errstr(cdb2_hndl_tp* hndl) except + int cdb2_bind_param(cdb2_hndl_tp *hndl, const char *name, int type, const void *varaddr, int length) except + + int cdb2_bind_index(cdb2_hndl_tp *hndl, int index, int type, const void *varaddr, int length) except + int cdb2_bind_array(cdb2_hndl_tp *hndl, const char *name, cdb2_coltype, const void *varaddr, size_t count, size_t typelen) except + + int cdb2_bind_array_index(cdb2_hndl_tp *hndl, int index, cdb2_coltype, const void *varaddr, size_t count, size_t typelen) except + int cdb2_clearbindings(cdb2_hndl_tp *hndl) except + int cdb2_clear_ack(cdb2_hndl_tp *hndl) except + diff --git a/comdb2/cdb2.py b/comdb2/cdb2.py index 01f8c7f..e64038a 100644 --- a/comdb2/cdb2.py +++ b/comdb2/cdb2.py @@ -77,9 +77,14 @@ ----------------- In real applications you'll often need to pass parameters into a SQL query. -This is done using parameter binding - in the query, placeholders are specified -using ``@name``, and a mapping of names to values is passed to `Handle.execute` -along with the query. For example: +This is done using parameter binding, either by name or by position. + +By Name +~~~~~~~ + +In the query, placeholders are specified using ``@name`` or ``:name``, and a +mapping of names to values is passed to `Handle.execute` along with the query. +For example: >>> query = "select 25 between @a and @b" >>> print(list(hndl.execute(query, {'a': 20, 'b': 42}))) @@ -88,16 +93,33 @@ >>> print(list(hndl.execute(query, params))) [[0]] -In this example, we run the query with two different sets of -parameters, producing different results. First, we execute -the query with ``@a`` bound to ``20`` and ``@b`` bound to ``42``. In this -case, because ``20 <= 25 <= 42``, the expression evaluates to true, and a ``1`` -is returned. +In this example, we run the query with two different sets of parameters, +producing different results. First, we execute the query with ``@a`` bound to +``20`` and ``@b`` bound to ``42``. In this case, because ``20 <= 25 <= 42``, +the expression evaluates to true, and a single row with a single column +set to ``1`` is returned. -When we run the same query with ``@b`` bound to ``23``, a ``0`` -is returned instead, because ``20 <= 25 <= 23`` is false. In both of these -examples we make use of the `list` constructor to turn the iterable returned -by `Handle.execute` into a list of result rows. +When we run the same query with ``@b`` bound to ``23``, a row with a column +set to ``0`` is returned instead, because ``20 <= 25 <= 23`` is false. + +Note: + In both of these examples we make use of the `list` constructor to turn + the iterable returned by `Handle.execute` into a list of result rows. + +By Position +~~~~~~~~~~~ + +You can bind parameters positionally rather than by name, by using ``?`` +for each placeholder and providing a list or tuple of parameter values. +For example: + + >>> query = "select 25 between ? and ?" + >>> print(list(hndl.execute(query, [20, 42]))) + [[1]] + +Here, we execute the query with the first ``?`` bound to 20 and the second +``?`` bound to 42, so a row with a single column set to ``1`` is returned +like in the previous example. Types ----- @@ -359,7 +381,7 @@ def row_factory( def execute( self, sql: str | bytes, - parameters: Mapping[str, ParameterValue] | None = None, + parameters: Mapping[str, ParameterValue] | Sequence[ParameterValue] | None = None, *, column_types: Sequence[ColumnType] | None = None, ) -> Handle: @@ -368,8 +390,10 @@ def execute( The ``sql`` string may have placeholders for parameters to be passed. This should always be the preferred method of parameterizing the SQL query, as it prevents SQL injection vulnerabilities and is faster. - Placeholders for named parameters must be in Comdb2's native format, - ``@param_name``. + Placeholders for named parameters must be in one of Comdb2's native + formats, either ``@param_name`` or ``:param_name``. Alternatively, you + can use ``?`` for each placeholder to bind parameters positionally + instead of by name. If ``column_types`` is provided and non-empty, it must be a sequence of members of the `ColumnType` enumeration. The database will coerce the @@ -381,8 +405,11 @@ def execute( Args: sql (str): The SQL string to execute. - parameters (Mapping[str, Any]): An optional mapping from parameter - names to the values to be bound for them. + parameters (Mapping[str, Any] | Sequence[Any]): + If the SQL statement has ``@param_name`` or ``:param_name`` + style placeholders, you must pass a mapping from parameter name + to value. If the SQL statement has ``?`` style placeholders, + you must instead pass an ordered sequence of parameter values. column_types (Sequence[int]): An optional sequence of types (values of the `ColumnType` enumeration) which the columns of the result set will be coerced to. @@ -399,7 +426,12 @@ def execute( Example: >>> for row in hndl.execute("select 1, 2 UNION ALL select @x, @y", - ... {'x': 2, 'y': 4}): + ... {"x": 2, "y": 4}): + ... print(row) + [1, 2] + [2, 4] + + >>> for row in hndl.execute("select 1, 2 UNION ALL select ?, ?", [2, 4]): ... print(row) [1, 2] [2, 4] diff --git a/comdb2/dbapi2.py b/comdb2/dbapi2.py index 581b685..e4ca04d 100644 --- a/comdb2/dbapi2.py +++ b/comdb2/dbapi2.py @@ -99,10 +99,15 @@ ----------------- In real applications you'll often need to pass parameters into a SQL query. -This is done using parameter binding - in the query, placeholders are specified -using ``%(name)s``, and a mapping of ``name`` to parameter value is passed to -`Cursor.execute` along with the query. The ``%(`` and ``)s`` are fixed, and -the ``name`` between them varies for each parameter. For example: +This is done using parameter binding, either by name or by position. + +By Name +~~~~~~~ + +In the query, placeholders are specified using ``%(name)s``, and a mapping of +``name`` to parameter value is passed to `Cursor.execute` along with the query. +The ``%(`` and ``)s`` are fixed, and the ``name`` inside them varies for each +parameter. For example: >>> query = "select 25 between %(a)s and %(b)s" >>> print(conn.cursor().execute(query, {'a': 20, 'b': 42}).fetchall()) @@ -114,17 +119,59 @@ In this example, we run the query with two different sets of parameters, producing different results. First, we execute the query with parameter ``a`` bound to ``20`` and ``b`` bound to ``42``. In this case, -because ``20 <= 25 <= 42``, the expression evaluates to true, and a ``1`` is -returned. +because ``20 <= 25 <= 42``, the expression evaluates to true, and a row +containing a single column with a value of ``1`` is returned. -When we run the same query with parameter ``b`` bound to ``23``, a ``0`` is -returned instead, because ``20 <= 25 <= 23`` is false. +When we run the same query with parameter ``b`` bound to ``23``, a row +containing a single column with a value of ``0`` is returned instead, because +``20 <= 25 <= 23`` is false. -Note: - Because parameters are bound using ``%(name)s``, other ``%`` signs in +Warning: + Because named parameters are bound using ``%(name)s``, other ``%`` signs in a query must be escaped. For example, ``WHERE name like 'M%'`` becomes ``WHERE name LIKE 'M%%'``. +Danger: + This applies even when no parameters are passed at all - passing ``None`` + or no parameters behaves the same as passing an empty `dict`. + +By Position +~~~~~~~~~~~ + +You can bind parameters positionally rather than by name, by using ``?`` +for each placeholder and providing a list or tuple of parameter values. +For example: + + >>> query = "select 25 between ? and ?" + >>> print(conn.cursor().execute(query, [20, 42]).fetchall()) + [[1]] + +In this example, we execute the query with the first ``?`` bound to 20 and the +second ``?`` bound to 42, so a row with a single column with a value of ``1`` +is returned like in the previous example. + +Warning: + Unlike when binding parameters by name, you must not escape ``%`` signs in + the SQL when binding parameters positionally. + + For example, you could do ``WHERE val % 5 = ?``, but not ``WHERE val %% 5 = ?``. + +Tip: + You can pass an empty tuple of parameters to avoid the need to escape ``%`` + signs in the SQL even when you don't want to bind any parameters, like: + + >>> query = "select 42 % 20" + >>> print(conn.cursor().execute(query, ()).fetchall()) + [[2]] + + Compare this against what happens if you don't pass any parameters at all: + + >>> print(conn.cursor().execute(query).fetchall()) + Traceback (most recent call last): + ... + comdb2.dbapi2.InterfaceError: Invalid Python format string for query + + Types ----- @@ -303,17 +350,31 @@ Comdb2's native placeholder format is ``@name``, but that cannot be used by this module because it's not an acceptable `DB-API 2.0 placeholder style `_. This module uses -``pyformat`` because it is the only DB-API 2.0 paramstyle that we can translate -into Comdb2's placeholder format without needing a SQL parser. +``pyformat`` for named parameters because it is the only DB-API 2.0 paramstyle +that we can translate into Comdb2's placeholder format without needing +a SQL parser. This module also supports the ``qmark`` parameter style for +binding parameters positionally. Note: An int value is bound as ``%(my_int)s``, not as ``%(my_int)d`` - the last character is always ``s``. Note: - Because SQL strings for this module use the ``pyformat`` placeholder style, - any literal ``%`` characters in a query must be escaped by doubling them. - ``WHERE name like 'M%'`` becomes ``WHERE name LIKE 'M%%'``. + When binding parameters by name, any ``%`` sign is recognized as the start + of a ``pyformat`` style placeholder, and so any literal ``%`` characters in + a SQL statement must be escaped by doubling. ``WHERE name like 'M%'`` + becomes ``WHERE name LIKE 'M%%'``. This does not apply when binding + parameters positionally with ``?`` placeholders, nor when the literal ``%`` + appears in a parameter value as opposed to literally in the query. + In either of those cases, the ``%`` characters must not be escaped. + +Warning: + Literal ``%`` signs in the query must be escaped when no parameters are + passed at all -- passing ``None`` or no parameters behaves the same as + passing an empty `dict`. You can avoid the need to escape ``%`` signs in + an unparametrized query by instead passing an empty `tuple` as parameters, + which causes the statement to be treated as having ``qmark`` placeholders + instead of ``pyformat`` placeholders. """ _FIRST_WORD_OF_STMT = re.compile( @@ -996,15 +1057,22 @@ def callproc(self, procname: str, parameters: Sequence[ParameterValue]) -> Seque def execute( self, sql: str, - parameters: Mapping[str, ParameterValue] | None = None, + parameters: Mapping[str, ParameterValue] | Sequence[ParameterValue] | None = None, *, column_types: Sequence[ColumnType] | None = None, ) -> Cursor: """Execute a database operation (query or command). - The ``sql`` string must be provided as a Python format string, with - parameter placeholders represented as ``%(name)s`` and all other ``%`` - signs escaped as ``%%``. + The ``sql`` string may contain either named placeholders represented + as ``%(name)s`` or positionally ordered placeholders represented + as ``?``. + + When the parameters are a mapping or ``None``, named placeholders are + being used, and so any literal ``%`` signs in the statement must be + escaped by doubling them, to distinguish them from the start of a named + placeholder. When a sequence of parameters is provided instead, any + placeholders must be positional ``?`` placeholders, and literal ``%`` + signs in the SQL must not be escaped. Note: Using placeholders should always be the preferred method of @@ -1027,8 +1095,11 @@ def execute( Args: sql (str): The SQL string to execute, as a Python format string. - parameters (Mapping[str, Any]): An optional mapping from parameter - names to the values to be bound for them. + parameters (Mapping[str, Any] | Sequence[Any]): + If the SQL statement has ``%(param_name)s`` style placeholders, + you must pass a mapping from parameter name to value. + If the SQL statement has ``?`` style placeholders, you must + instead pass an ordered sequence of parameter values. column_types (Sequence[int]): An optional sequence of types (values of the `ColumnType` enumeration) which the columns of the result set will be coerced to. @@ -1049,6 +1120,10 @@ def execute( ... {'x': 2, 'y': 4}) >>> cursor.fetchall() [[1, 2], [2, 4]] + + >>> cursor.execute("select 1, 2 UNION ALL select ?, ?", [2, 4]]) + >>> cursor.fetchall() + [[1, 2], [2, 4]] """ self._check_closed() self._description = None @@ -1068,7 +1143,7 @@ def execute( return self def executemany( - self, sql: str, seq_of_parameters: Sequence[Mapping[str, ParameterValue]] + self, sql: str, seq_of_parameters: Sequence[Mapping[str, ParameterValue]] | Sequence[Sequence[ParameterValue]] ) -> None: """Execute the same SQL statement repeatedly with different parameters. @@ -1078,10 +1153,10 @@ def executemany( Args: sql (str): The SQL string to execute, as a Python format string of the format expected by `execute`. - seq_of_parameters (Sequence[Mapping[str, Any]]): A sequence of - mappings from parameter names to the values to be bound for - them. The ``sql`` statement will be run once per element in - this sequence. + seq_of_parameters (Sequence[Mapping[str, Any]] | Sequence[Sequence[Any]]): + The ``sql`` statement will be executed once per element in this + sequence, using each successive element as the parameter values + for the corresponding call to `.execute`. """ self._check_closed() for parameters in seq_of_parameters: @@ -1105,7 +1180,10 @@ def _execute(self, operation, sql, parameters=None, *, column_types=None): try: # If variable interpolation fails, then translate the exception to # an InterfaceError to signal that it's a client-side problem. - sql = sql % {name: "@" + name for name in parameters} + # If binding by index then no need to modify sql + by_name = hasattr(parameters, "items") + if by_name: + sql = sql % {name: "@" + name for name in parameters} except KeyError as keyerr: msg = "No value provided for parameter %s" % keyerr raise InterfaceError(msg) from keyerr diff --git a/docs/tips.rst b/docs/tips.rst index ba12033..d2ba8c4 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -68,14 +68,29 @@ Best Practices, Tips, and Tricks placeholders. See `.dbapi2.Cursor.execute` and `.cdb2.Handle.execute` for details. -#. For `.dbapi2`, be sure to escape any ``%`` signs in a query by doubling - them. That is, instead of:: +#. When using `.dbapi2` but not parameter binding by position be sure to escape + any literal ``%`` signs in a query by doubling them. That is, instead of:: - c.execute("select * from tbl where col like 'foo%'") + cursor.execute( + "select * from log where msg like 'ERROR: %' and pid = %(pid)s", + {"pid": 1234}, + ) You need to write:: - c.execute("select * from tbl where col like 'foo%%'") + cursor.execute( + "select * from log where msg like 'ERROR: %%' and pid = %(pid)s", + {"pid": 1234}, + ) + + Also, if you don't want to pass any parameters to a `.dbapi2` query, it's + better to pass ``()`` than the default ``None`` value, because this removes + the need to escape any literal ``%`` signs that appear in the query:: + + cursor.execute( + "select * from log where msg like 'ERROR: %'", + (), + ) See `.dbapi2.paramstyle` for an explanation of why. diff --git a/tests/test_cdb2.py b/tests/test_cdb2.py index da8b736..0fb7abe 100644 --- a/tests/test_cdb2.py +++ b/tests/test_cdb2.py @@ -62,8 +62,26 @@ def test_binding_parameters(): hndl.execute("insert into simple(key, val) values(@k, @v)", dict(k=3, v=4)) assert hndl.get_effects()[0] == 1 + hndl.execute("insert into simple(key, val) values(?, ?)", [5, 6]) + assert hndl.get_effects()[0] == 1 + rows = list(hndl.execute("select key, val from simple order by key")) - assert rows == [[1, 2], [3, 4]] + assert rows == [[1, 2], [3, 4], [5, 6]] + + +def test_binding_no_parameters(): + hndl = cdb2.Handle("mattdb", "dev") + rows = list(hndl.execute("select 1 % 10")) + assert rows == [[1]] + + rows = list(hndl.execute("select 2 % 10", {})) + assert rows == [[2]] + + rows = list(hndl.execute("select 3 % 10", ())) + assert rows == [[3]] + + rows = list(hndl.execute("select 4 % 10", [])) + assert rows == [[4]] def test_commit_failures(): diff --git a/tests/test_dbapi2.py b/tests/test_dbapi2.py index 3dbd175..4f79b3e 100644 --- a/tests/test_dbapi2.py +++ b/tests/test_dbapi2.py @@ -28,6 +28,7 @@ DataError, Datetime, DatetimeUs, + Error, ForeignKeyConstraintError, IntegrityError, InterfaceError, @@ -256,12 +257,51 @@ def test_extra_percent_arg(): cursor.execute("insert into simple(key, val) values(%(k)s, %(v)s)", dict(k=3)) -def test_unescaped_percent(): +def test_binding_no_parameters(): conn = connect("mattdb", "dev") cursor = conn.cursor() - cursor.execute("select 1%%2") # Should work + + rows = list(cursor.execute("select 1 %% 10")) + assert rows == [[1]] + + rows = list(cursor.execute("select 2 %% 10", {})) + assert rows == [[2]] + + rows = list(cursor.execute("select 3 % 10", ())) + assert rows == [[3]] + + rows = list(cursor.execute("select 4 % 10", [])) + assert rows == [[4]] + + +def test_incorrect_escaping_of_percent_signs(): + conn = connect("mattdb", "dev") + cursor = conn.cursor() + + with pytest.raises(InterfaceError): + cursor.execute("select 1 % 10") + with pytest.raises(InterfaceError): - cursor.execute("select 1%2") + cursor.execute("select 2 % 10", {}) + + with pytest.raises(ProgrammingError): + cursor.execute("select 3 %% 10", ()) + + with pytest.raises(ProgrammingError): + cursor.execute("select 4 %% 10", []) + + +def test_binding_different_sequence_types(): + conn = connect("mattdb", "dev") + cursor = conn.cursor() + cursor.execute("select ?, ?", [1, 2]) + assert cursor.fetchall() == [[1, 2]] + + cursor.execute("select ?, ?", (1, 2)) + assert cursor.fetchall() == [[1, 2]] + + cursor.execute("select ?, ?", "hi") + assert cursor.fetchall() == [["h", "i"]] def test_reading_and_writing_datetimes(): @@ -377,6 +417,22 @@ def test_all_datatypes_as_parameters(): assert row == list(v for k, v in params) assert cursor.fetchone() is None + cursor.execute("delete from all_datatypes") + conn.commit() + + cursor.execute( + "insert into all_datatypes(" + ", ".join(COLUMN_LIST) + ")" + " values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + [v for _, v in params], + ) + + conn.commit() + + cursor.execute("select * from all_datatypes") + row2 = cursor.fetchone() + assert row2 == row + assert cursor.fetchone() is None + def test_naive_datetime_as_parameter(): conn = connect("mattdb", "dev") @@ -414,6 +470,22 @@ def test_naive_datetime_as_parameter(): assert row == [Datetime(2009, 2, 13, 18, 31, 30, 234000, pytz.UTC)] assert cursor.fetchone() is None + cursor.execute("delete from all_datatypes") + conn.commit() + + cursor.execute( + "insert into all_datatypes(" + ", ".join(COLUMN_LIST) + ")" + " values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + list(v for k, v in params), + ) + + conn.commit() + + cursor.execute("select datetime_col from all_datatypes") + row2 = cursor.fetchone() + assert row2 == row + assert cursor.fetchone() is None + def test_datetime_with_non_olson_tzname(): conn = connect("mattdb", "dev") @@ -431,6 +503,12 @@ def test_datetime_with_non_olson_tzname(): assert row[1].tzname() == "UTC" assert row[1] == edt_dt + row = cursor.execute("select ?, ?", [est_dt, edt_dt]).fetchone() + assert row[0].tzname() == "UTC" + assert row[0] == est_dt + assert row[1].tzname() == "UTC" + assert row[1] == edt_dt + def test_rounding_datetime_to_nearest_millisecond(): conn = connect("mattdb", "dev") @@ -443,10 +521,16 @@ def test_rounding_datetime_to_nearest_millisecond(): cursor.execute("select @date", {"date": curr_microsecond}) assert cursor.fetchall() == [[prev_millisecond]] + cursor.execute("select ?", [curr_microsecond]) + assert cursor.fetchall() == [[prev_millisecond]] + curr_microsecond += datetime.timedelta(microseconds=1) cursor.execute("select @date", {"date": curr_microsecond}) assert cursor.fetchall() == [[next_millisecond]] + cursor.execute("select ?", [curr_microsecond]) + assert cursor.fetchall() == [[next_millisecond]] + def test_cursor_description(): conn = connect("mattdb", "dev") @@ -501,6 +585,9 @@ def test_binding_number_that_overflows_long_long(): with pytest.raises(DataError): cursor.execute("select @i", dict(i=2**64 + 1)) + with pytest.raises(DataError): + cursor.execute("select ?", [2**64 + 1]) + def test_retrieving_null(): conn = connect("mattdb", "dev") @@ -924,6 +1011,13 @@ def test_parameter_binding_arrays(values): cursor.execute("select * from carray(%(values)s)", dict(values=values)) results = cursor.fetchall() + # THEN + assert results == [[v] for v in values] + + # WHEN + cursor.execute("select * from carray(?)", [values]) + results = cursor.fetchall() + # THEN assert results == [[v] for v in values] conn.close() @@ -934,27 +1028,27 @@ def test_parameter_binding_arrays(values): [ ( [], - "Can't bind list value [] for parameter 'values': " + "Can't bind list value [] for parameter {}: " + "ValueError: empty lists cannot be bound", ), ( (), - "Can't bind tuple value () for parameter 'values': " + "Can't bind tuple value () for parameter {}: " + "ValueError: empty tuples cannot be bound", ), ( [1, "hello"], - "Can't bind list value [1, 'hello'] for parameter 'values': " + "Can't bind list value [1, 'hello'] for parameter {}: " + "ValueError: all list elements must be the same type", ), ( (1j,), - "Can't bind tuple value (1j,) for parameter 'values': " + "Can't bind tuple value (1j,) for parameter {}: " + "ValueError: Cannot bind a tuple of complex", ), ( ((1, 2, 3),), - "Can't bind tuple value ((1, 2, 3),) for parameter 'values': " + "Can't bind tuple value ((1, 2, 3),) for parameter {}: " + "ValueError: Cannot bind a tuple of tuple", ), ], @@ -965,9 +1059,13 @@ def test_parameter_binding_invalid_arrays(values, exc_msg): cursor = conn.cursor() # WHEN/THEN - with pytest.raises(DataError, match=re.escape(exc_msg)): + with pytest.raises(DataError, match=re.escape(exc_msg.format("'values'"))): cursor.execute("select * from carray(%(values)s)", dict(values=values)) + # WHEN/THEN + with pytest.raises(DataError, match=re.escape(exc_msg.format("1"))): + cursor.execute("select * from carray(?)", [values]) + def test_specifying_column_types(): # GIVEN From 1a792fc8b8d32b5fb6f4a49b0249fbaf9c984d7f Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Mon, 24 Mar 2025 11:58:05 -0400 Subject: [PATCH 40/62] Release version 1.8.0 This release adds support for binding parameters positionally with `?` placeholders, and adds a new `.as_datetime` method to `DatetimeUs` to allow converting a `DatetimeUs` to a `datetime.datetime`. This release also drops support for Python 3.7, which has at this point is several years past end-of-life, and which has become too difficult to support in our CI. Signed-off-by: Matt Wozniski --- comdb2/_about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comdb2/_about.py b/comdb2/_about.py index 8763b29..0eb9fd7 100644 --- a/comdb2/_about.py +++ b/comdb2/_about.py @@ -1,2 +1,2 @@ __all__ = ["__version__"] -__version__ = "1.7.2" +__version__ = "1.8.0" From f3a4419314fe5a8ba620dbdccee373e7d742d071 Mon Sep 17 00:00:00 2001 From: Michael Thorpe Date: Mon, 21 Jul 2025 12:32:59 -0400 Subject: [PATCH 41/62] tests: Ensure `dbapi2.Error` sets `__cause__` Existing users rely on this to access the underlying Comdb2 error code. Signed-off-by: Michael Thorpe --- .gitignore | 1 + tests/test_dbapi2.py | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index ba3a04f..e80a260 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /build +/comdb2.egg-info/ /dist /python_comdb2.egg-info *.pyc diff --git a/tests/test_dbapi2.py b/tests/test_dbapi2.py index 4f79b3e..f3178be 100644 --- a/tests/test_dbapi2.py +++ b/tests/test_dbapi2.py @@ -28,7 +28,6 @@ DataError, Datetime, DatetimeUs, - Error, ForeignKeyConstraintError, IntegrityError, InterfaceError, @@ -793,6 +792,21 @@ def raise_not_supported_error(): conn.commit() +@patch("comdb2.cdb2.Handle") +def test_error_wraps_underlying(handle): + handle.return_value.execute.side_effect = partial(throw_on, "begin") + + conn = connect("mattdb", "dev") + cursor = conn.cursor() + + with pytest.raises(OperationalError) as exc_info: + cursor.execute("insert into simple(key, val) values(1, 2)") + + assert isinstance(exc_info.value.__cause__, cdb2.Error) + underlying = exc_info.value.__cause__ + assert underlying.error_code == 42 + + def test_autocommit_handles(): conn = connect("mattdb", "dev", autocommit=True) cursor = conn.cursor() @@ -936,8 +950,7 @@ def test_unsupported_column_decode_exception(): cursor.fetchall() errmsg = ( - "Failed to decode CDB2_INTERVALDS column 0 ('delta'):" - " Unsupported column type" + "Failed to decode CDB2_INTERVALDS column 0 ('delta'): Unsupported column type" ) assert errmsg in str(exc_info.value) From 78acb3930271bca84cb01ec2e8eef7e588bca653 Mon Sep 17 00:00:00 2001 From: Jonathan Surany Date: Tue, 1 Apr 2025 15:23:20 -0400 Subject: [PATCH 42/62] Fix annotation for `dbapi2.Cursor.description` This attribute can be `None`, for instance on a newly created `Cursor` or one where a transaction has just been committed or rolled back. Signed-off-by: Jonathan Surany --- comdb2/dbapi2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comdb2/dbapi2.py b/comdb2/dbapi2.py index e4ca04d..26a057f 100644 --- a/comdb2/dbapi2.py +++ b/comdb2/dbapi2.py @@ -936,7 +936,7 @@ def arraysize(self, value: int) -> None: @property def description( self, - ) -> tuple[tuple[str, object, None, None, None, None, None], ...]: + ) -> tuple[tuple[str, object, None, None, None, None, None], ...] | None: """Provides the name and type of each column in the latest result set. This read-only attribute will contain one element per column in the From 2ef8f03d18d16693dab5e82871b4fb6a765310ff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 17:57:52 +0000 Subject: [PATCH 43/62] Bump actions/checkout from 4 to 5 Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- .github/workflows/docs.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d3de70e..b7a902b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout bloomberg-comdb2 repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: bloomberg/comdb2 path: bloomberg-comdb2 @@ -84,7 +84,7 @@ jobs: find /opt/bb && rm bloomberg-comdb2.tar.gz ' - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: 'Set up Python 3.8' uses: actions/setup-python@v5 # note that this step overwrites the PKG_CONFIG_PATH variable with: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 05b3bec..5bf2eee 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -20,7 +20,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python uses: actions/setup-python@v5 with: From eca7ccc815931f451b9e203c811c49affbe9d956 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 21:18:25 +0000 Subject: [PATCH 44/62] Bump actions/download-artifact from 4 to 5 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 4 to 5. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b7a902b..d05b893 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -73,7 +73,7 @@ jobs: needs: [build_bloomberg_comdb2] steps: - name: Download bloomberg-comdb2 with build artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v5 with: name: bloomberg-comdb2 path: . @@ -116,7 +116,7 @@ jobs: - '3.8' steps: - name: Download comdb2 repo with build artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v5 with: name: bloomberg-comdb2 path: . @@ -148,7 +148,7 @@ jobs: done ' - name: Download python-comdb2 sdist - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v5 with: name: python-comdb2-sdist path: dist From b1a8028a54fcab341b6cea5b7b078ae4a280280c Mon Sep 17 00:00:00 2001 From: Ivona Stojanovic Date: Wed, 13 Aug 2025 11:03:12 +0100 Subject: [PATCH 45/62] Add Python 3.14 support We'll use 3.14-dev for now, as 3.14.0 has not been released yet. Signed-off-by: Ivona Stojanovic --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d05b893..bc1cc90 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -108,6 +108,7 @@ jobs: strategy: matrix: python-version: + - '3.14-dev' - '3.13' - '3.12' - '3.11' From 69fa93d6fe0d46c3b19d80af650e69bdeb44cdcb Mon Sep 17 00:00:00 2001 From: Ivona Stojanovic Date: Wed, 13 Aug 2025 11:03:53 +0100 Subject: [PATCH 46/62] Fix test compatibility with Python 3.14 error message changes Update test_date_column_decode_exception to handle both old and new datetime year validation error message formats. Python 3.14 changed the error message from "year 0 is out of range" to "year must be in MINYEAR..MAXYEAR, not 0" for invalid year values. The test now checks for either format to maintain compatibility across Python versions. Signed-off-by: Ivona Stojanovic --- tests/test_dbapi2.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_dbapi2.py b/tests/test_dbapi2.py index f3178be..f9a9c11 100644 --- a/tests/test_dbapi2.py +++ b/tests/test_dbapi2.py @@ -937,7 +937,11 @@ def test_date_column_decode_exception(): exc_str = str(exc_info.value) assert "Failed to decode CDB2_DATETIME column 0 ('date'):" in exc_str - assert " out of range" in exc_str + # Check for both old and new error message formats + # Python < 3.14: "year 0 is out of range" + # Python >= 3.14: "year must be in MINYEAR..MAXYEAR, not 0" + # See: https://github.com/python/cpython/commit/3e222e3a15959690a41847a1177ac424427815e5#diff-3ee250e3806e884518fd872e9148baf532de6ec54c1cdb4e7679fbb2869d9c47 + assert (" out of range" in exc_str or "year must be in" in exc_str) def test_unsupported_column_decode_exception(): From 54113569164a4e784cf016dcebea19d667831a61 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Wed, 13 Aug 2025 21:02:00 -0400 Subject: [PATCH 47/62] tests: Drop dependency on `pytz` Signed-off-by: Matt Wozniski --- tests/test_cdb2_datetimeus.py | 14 ++++++------- tests/test_dbapi2.py | 38 ++++++++++++++++------------------- 2 files changed, 23 insertions(+), 29 deletions(-) diff --git a/tests/test_cdb2_datetimeus.py b/tests/test_cdb2_datetimeus.py index 334ea56..c611fb6 100644 --- a/tests/test_cdb2_datetimeus.py +++ b/tests/test_cdb2_datetimeus.py @@ -11,7 +11,6 @@ from comdb2 import cdb2 -import pytz import dateutil.tz import pytest @@ -84,11 +83,11 @@ def test_datetimeus_fromtimestamp(): def test_datetimeus_astimezone(): - eastern = pytz.timezone("US/Eastern") - loc_dt = eastern.localize(datetime.datetime(2002, 10, 27, 6, 0, 0)) + eastern = dateutil.tz.gettz("US/Eastern") + loc_dt = datetime.datetime(2002, 10, 27, 6, 0, 0, tzinfo=eastern) dtu = cdb2.DatetimeUs.fromdatetime(loc_dt) - assert type(dtu.astimezone(pytz.utc)) == cdb2.DatetimeUs + assert type(dtu.astimezone(datetime.timezone.utc)) == cdb2.DatetimeUs def test_datetimeus_as_datetime_naive(): @@ -99,7 +98,7 @@ def test_datetimeus_as_datetime_naive(): def test_datetimeus_as_datetime_with_tz(): - dtu = cdb2.DatetimeUs.fromtimestamp(time.time(), tz=pytz.UTC) + dtu = cdb2.DatetimeUs.fromtimestamp(time.time(), tz=datetime.timezone.utc) dt = dtu.as_datetime() assert type(dt) == datetime.datetime assert dt == dtu @@ -109,10 +108,9 @@ def test_datetimeus_type_stickiness(): def check(obj): assert isinstance(obj, cdb2.DatetimeUs) - new_york = pytz.timezone("America/New_York") - utc = pytz.UTC + new_york = dateutil.tz.gettz("America/New_York") + utc = datetime.timezone.utc - check(new_york.localize(cdb2.DatetimeUs(2016, 8, 15, 18, 47, 15, 123456))) check(cdb2.DatetimeUs(2016, 8, 15, 18, 47, 15, 123456, new_york)) check(cdb2.DatetimeUs(2016, 8, 15, 18, 47, 15, 123456)) check(cdb2.DatetimeUs.today()) diff --git a/tests/test_dbapi2.py b/tests/test_dbapi2.py index f9a9c11..a1cbd6e 100644 --- a/tests/test_dbapi2.py +++ b/tests/test_dbapi2.py @@ -16,7 +16,7 @@ from unittest.mock import patch import pytest -import pytz +from dateutil.tz import gettz from comdb2 import cdb2 from comdb2.dbapi2 import ( @@ -306,7 +306,7 @@ def test_binding_different_sequence_types(): def test_reading_and_writing_datetimes(): conn = connect("mattdb", "dev") cursor = conn.cursor() - ts_obj = Timestamp(2015, 1, 2, 3, 4, 5, 123000, pytz.UTC) + ts_obj = Timestamp(2015, 1, 2, 3, 4, 5, 123000, datetime.timezone.utc) ts_str_in = "2015-01-02T03:04:05.12345" ts_str_out = "2015-01-02T030405.123 UTC" @@ -320,7 +320,7 @@ def test_reading_and_writing_datetimes(): def test_reading_and_writing_datetimeus(): conn = connect("mattdb", "dev") cursor = conn.cursor() - ts_obj = TimestampUs(2015, 1, 2, 3, 4, 5, 123456, pytz.UTC) + ts_obj = TimestampUs(2015, 1, 2, 3, 4, 5, 123456, datetime.timezone.utc) ts_str_in = "2015-01-02T03:04:05.123456" ts_str_out = "2015-01-02T030405.123456 UTC" @@ -367,9 +367,7 @@ def test_inserting_one_row_with_all_datatypes_without_parameters(): "hello", "goodbye", Binary("\x01\x02\x03\x04\x05\x06\x07"), - pytz.timezone("America/New_York").localize( - Datetime(2009, 2, 13, 18, 31, 30, 234000) - ), + Datetime(2009, 2, 13, 18, 31, 30, 234000, gettz("America/New_York")), "hello world", ] assert cursor.fetchone() is None @@ -393,9 +391,7 @@ def test_all_datatypes_as_parameters(): ("blob_col", Binary("")), ( "datetime_col", - pytz.timezone("America/New_York").localize( - Datetime(2009, 2, 13, 18, 31, 30, 234000) - ), + Datetime(2009, 2, 13, 18, 31, 30, 234000, gettz("America/New_York")) ), ("vutf8_col", "foo" * 50), ) @@ -466,7 +462,7 @@ def test_naive_datetime_as_parameter(): cursor.connection.commit() cursor.execute("select datetime_col from all_datatypes") row = cursor.fetchone() - assert row == [Datetime(2009, 2, 13, 18, 31, 30, 234000, pytz.UTC)] + assert row == [Datetime(2009, 2, 13, 18, 31, 30, 234000, datetime.timezone.utc)] assert cursor.fetchone() is None cursor.execute("delete from all_datatypes") @@ -489,33 +485,33 @@ def test_naive_datetime_as_parameter(): def test_datetime_with_non_olson_tzname(): conn = connect("mattdb", "dev") cursor = conn.cursor() - tz = pytz.timezone("America/New_York") + tz = gettz("America/New_York") dt = datetime.datetime(2016, 11, 6, 1, 30, 0, 123000) - est_dt = tz.localize(dt, is_dst=False) - edt_dt = tz.localize(dt, is_dst=True) + est_dt = dt.replace(tzinfo=tz, fold=1) + edt_dt = dt.replace(tzinfo=tz, fold=0) assert est_dt.tzname() == "EST" assert edt_dt.tzname() == "EDT" params = {"est_dt": est_dt, "edt_dt": edt_dt} row = cursor.execute("select @est_dt, @edt_dt", params).fetchone() assert row[0].tzname() == "UTC" - assert row[0] == est_dt + assert row[0] == est_dt.astimezone(datetime.timezone.utc) assert row[1].tzname() == "UTC" - assert row[1] == edt_dt + assert row[1] == edt_dt.astimezone(datetime.timezone.utc) row = cursor.execute("select ?, ?", [est_dt, edt_dt]).fetchone() assert row[0].tzname() == "UTC" - assert row[0] == est_dt + assert row[0] == est_dt.astimezone(datetime.timezone.utc) assert row[1].tzname() == "UTC" - assert row[1] == edt_dt + assert row[1] == edt_dt.astimezone(datetime.timezone.utc) def test_rounding_datetime_to_nearest_millisecond(): conn = connect("mattdb", "dev") cursor = conn.cursor() - curr_microsecond = Datetime(2016, 2, 28, 23, 59, 59, 999499, pytz.UTC) - prev_millisecond = Datetime(2016, 2, 28, 23, 59, 59, 999000, pytz.UTC) - next_millisecond = Datetime(2016, 2, 29, 0, 0, 0, 0, pytz.UTC) + curr_microsecond = Datetime(2016, 2, 28, 23, 59, 59, 999499, datetime.timezone.utc) + prev_millisecond = Datetime(2016, 2, 28, 23, 59, 59, 999000, datetime.timezone.utc) + next_millisecond = Datetime(2016, 2, 29, 0, 0, 0, 0, datetime.timezone.utc) cursor.execute("select @date", {"date": curr_microsecond}) assert cursor.fetchall() == [[prev_millisecond]] @@ -964,7 +960,7 @@ def test_datetimeus(): hndl = connect("mattdb", "dev") cursor = hndl.cursor() - sent = DatetimeUs(2017, 8, 16, 19, 32, 2, 825022, tzinfo=pytz.UTC) + sent = DatetimeUs(2017, 8, 16, 19, 32, 2, 825022, tzinfo=datetime.timezone.utc) cursor.execute(query, dict(date=sent)) rcvd = cursor.fetchall()[0][0] From 70fc9c7df74f0aa2e4143aefcb903614fd520b4b Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Thu, 14 Aug 2025 17:57:13 -0400 Subject: [PATCH 48/62] docs: Clarify that we treat naive datetimes as UTC This originally was not the case, but this behavior was accidentally changed c0f99cf6cf351adfa4de67e4a9048e1b7f3f58c9 way back in 2017 while fixing a bug in our handling of aware datetimes, and the regression wasn't noticed. At this point Hyrum's Law tells us that the new behavior is almost certainly depended upon by someone and fixing the bug would be far riskier than just updating the documentation to match the current behavior. Signed-off-by: Matt Wozniski --- docs/types.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/types.rst b/docs/types.rst index 9514526..d452022 100644 --- a/docs/types.rst +++ b/docs/types.rst @@ -91,9 +91,8 @@ Whenever either type of datetime is retrieved as a result column, the returned `datetime.datetime` object will be aware, meaning it will be associated with a timezone (namely, the current timezone of the Comdb2 connection). Whenever either type of datetime is sent as a query parameter, it is permitted to be -naïve, in which case it uses the current timezone of the Comdb2 connection. By -default that timezone is UTC but it can be changed on a per-connection basis if -needed. +naïve, in which case it is treated as UTC (and *not* as the system's local +time, which is the more common interpretation in newer Python libraries). Comdb2 additionally supports three types for representing time intervals. One represents an exact number of elapsed milliseconds between two times From d6cdd2845cadcc4e03f58c29aa887e5527d44738 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Thu, 14 Aug 2025 17:08:02 -0400 Subject: [PATCH 49/62] Fix binding of ambiguous datetimes with `fold=1` When binding a `datetime.datetime` (but _not_ a `DatetimeUs`), the `fold` attribute was incorrectly ignored, leading to the first occurrence of the given wall clock time being bound, rather than the later occurrence. This was caused by the binding layer rounding the parameter value to the nearest millisecond by adding a `timedelta` to it, which resets the `fold` attribute to 0. Work around this by converting to UTC first, before rounding to the nearest millisecond. This works since there are no ambiguous times in UTC and so the `fold` attribute must always be zero. Signed-off-by: Matt Wozniski --- comdb2/_ccdb2.pyx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/comdb2/_ccdb2.pyx b/comdb2/_ccdb2.pyx index 43b0bae..23db3fe 100644 --- a/comdb2/_ccdb2.pyx +++ b/comdb2/_ccdb2.pyx @@ -82,6 +82,11 @@ cdef _describe_exception(exc): cdef _bind_datetime(obj, client_datetime *val): + if obj.tzinfo is None: + obj = obj.replace(tzinfo=datetime.timezone.utc) + else: + obj = obj.astimezone(datetime.timezone.utc) + if client_datetime is lib.cdb2_client_datetimeus_t: val.usec = obj.microsecond else: From 3b96cffc52f8e34ca1ed0012db01fdd5a8471916 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Thu, 14 Aug 2025 18:12:20 -0400 Subject: [PATCH 50/62] Stop using pytz from Python 3.14 onwards I'm a bit nervous about pulling the trigger on getting rid of `pytz` entirely, so this is a baby step in the right direction: starting from Python 3.14, use `zoneinfo` to create the time zone objects that we attach to the `datetime` objects fetched by queries, rather than `pytz`. Signed-off-by: Matt Wozniski --- comdb2/_ccdb2.pyx | 32 ++++++++++++++++++++++++-------- setup.py | 2 +- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/comdb2/_ccdb2.pyx b/comdb2/_ccdb2.pyx index 23db3fe..e0de1ad 100644 --- a/comdb2/_ccdb2.pyx +++ b/comdb2/_ccdb2.pyx @@ -20,8 +20,7 @@ from cpython.ref cimport Py_TYPE, PyObject from cpython.mem cimport PyMem_Malloc, PyMem_Free import datetime - -from pytz import timezone, UTC +import sys from ._cdb2_types import Error, Effects, DatetimeUs from . cimport _cdb2api as lib @@ -266,11 +265,13 @@ cdef _make_datetime(client_datetime *val): val.tm.tm_min, val.tm.tm_sec, usec, - UTC, + datetime.timezone.utc, pytype) - return timezone(val.tzname).localize( - PyDateTimeAPI.DateTime_FromDateAndTime( + if sys.version_info >= (3, 14): + from zoneinfo import ZoneInfo + + return PyDateTimeAPI.DateTime_FromDateAndTimeAndFold( val.tm.tm_year + 1900, val.tm.tm_mon + 1, val.tm.tm_mday, @@ -278,9 +279,24 @@ cdef _make_datetime(client_datetime *val): val.tm.tm_min, val.tm.tm_sec, usec, - None, - pytype), - val.tm.tm_isdst) + ZoneInfo(val.tzname), + not val.tm.tm_isdst, + pytype) + else: + from pytz import timezone + + return timezone(val.tzname).localize( + PyDateTimeAPI.DateTime_FromDateAndTime( + val.tm.tm_year + 1900, + val.tm.tm_mon + 1, + val.tm.tm_mday, + val.tm.tm_hour, + val.tm.tm_min, + val.tm.tm_sec, + usec, + None, + pytype), + val.tm.tm_isdst) cdef _column_value(lib.cdb2_hndl_tp *hndl, int col): diff --git a/setup.py b/setup.py index fe7def5..f0fd474 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ def make_static_extension(name, **kwargs): author="Alex Chamberlain", author_email="achamberlai9@bloomberg.net", packages=["comdb2"], - install_requires=["pytz"], + install_requires=["pytz; python_version < '3.14'"], extras_require={"tests": ["python-dateutil>=2.6.0", "pytest"]}, python_requires=">=3.8", ext_modules=[ccdb2], From 5f8dfd8054586c0f4a04349c6e72d20b414b1275 Mon Sep 17 00:00:00 2001 From: Sarah Monod Date: Thu, 14 Aug 2025 11:51:54 +0200 Subject: [PATCH 51/62] Release version 1.9.0 This release adds support for 3.14 Signed-off-by: Sarah Monod --- comdb2/_about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comdb2/_about.py b/comdb2/_about.py index 0eb9fd7..d9c8367 100644 --- a/comdb2/_about.py +++ b/comdb2/_about.py @@ -1,2 +1,2 @@ __all__ = ["__version__"] -__version__ = "1.8.0" +__version__ = "1.9.0" From ea7fe1ad1166b7f0ce1d55161282eb991e824552 Mon Sep 17 00:00:00 2001 From: Sarah Monod Date: Tue, 2 Sep 2025 10:53:16 -0400 Subject: [PATCH 52/62] ci: Install `tcl-dev` when building Comdb2 The `tcl-dev` build dependency used to be optional, and it has recently become required unconditionally. See https://github.com/bloomberg/comdb2/pull/5348/commits/f3ba9711fc6e61b4e8eee2287d45726d6128545f Signed-off-by: Sarah Monod --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bc1cc90..a05a0a7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,6 +46,7 @@ jobs: ncurses-dev protobuf-c-compiler tcl + tcl-dev uuid-dev zlib1g-dev ' From 97f0be73816016f3e5ae074d482c8b89cb1dcbeb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 13:16:46 +0000 Subject: [PATCH 53/62] Bump actions/download-artifact from 5 to 6 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 5 to 6. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a05a0a7..0ccffdb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -74,7 +74,7 @@ jobs: needs: [build_bloomberg_comdb2] steps: - name: Download bloomberg-comdb2 with build artifacts - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v6 with: name: bloomberg-comdb2 path: . @@ -118,7 +118,7 @@ jobs: - '3.8' steps: - name: Download comdb2 repo with build artifacts - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v6 with: name: bloomberg-comdb2 path: . @@ -150,7 +150,7 @@ jobs: done ' - name: Download python-comdb2 sdist - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v6 with: name: python-comdb2-sdist path: dist From cf2e6ecec0ea20864e8d7a134338f41f052fb00e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 13:10:00 +0000 Subject: [PATCH 54/62] Bump actions/upload-artifact from 4 to 5 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4 to 5. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0ccffdb..3dc2c93 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -63,7 +63,7 @@ jobs: - name: Archive bloomberg-comdb2 repo with build artifacts run: 'tar -C /opt -czvf bloomberg-comdb2.tar.gz bb' - name: Upload built bloomberg-comdb2 - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: bloomberg-comdb2 path: ./bloomberg-comdb2.tar.gz @@ -97,7 +97,7 @@ jobs: pipx run build --sdist ' - name: Upload sdist - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: python-comdb2-sdist path: dist/*.tar.gz From 10c73f2a7e82cdaeb6dac98e069f4eeb692f408b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Sep 2025 14:22:06 +0000 Subject: [PATCH 55/62] Bump actions/setup-python from 5 to 6 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5 to 6. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- .github/workflows/docs.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3dc2c93..ca0d069 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -87,7 +87,7 @@ jobs: ' - uses: actions/checkout@v5 - name: 'Set up Python 3.8' - uses: actions/setup-python@v5 # note that this step overwrites the PKG_CONFIG_PATH variable + uses: actions/setup-python@v6 # note that this step overwrites the PKG_CONFIG_PATH variable with: python-version: '3.8' # the lowest version that we support in CI - name: Build sdist @@ -169,7 +169,7 @@ jobs: /opt/bb/bin/cdb2sql "$COMDB2_DBNAME" local "create table $table_name { $(cat $table_file) }" done - name: Set up Python ${{matrix.python-version}} - uses: actions/setup-python@v5 # note that this step overwrites the PKG_CONFIG_PATH variable + uses: actions/setup-python@v6 # note that this step overwrites the PKG_CONFIG_PATH variable with: python-version: "${{matrix.python-version}}" - name: Install python-comdb2 from the sdist diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 5bf2eee..c58a11f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -22,7 +22,7 @@ jobs: steps: - uses: actions/checkout@v5 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.12" - name: Install Python dependencies From 96ce6ce4436c150b218776dd652632dafa215f8e Mon Sep 17 00:00:00 2001 From: Sarah Monod Date: Thu, 6 Nov 2025 11:28:58 -0500 Subject: [PATCH 56/62] Mark cursor drain test as such Not fully draining a cursor is usually a programming error, and we want to avoid it from happening. To that end, alarms have been setup and caught this particular test as being an offender. However, in this specific scenario, we do want to leave the cursor undrained, because this is precisely the behavior that we want to test, especially where it relates to the lifecycle of the underlying connection and garbage collection. Therefore, we are marking this test as a false positive to avoid having the alarm raised for this code. Signed-off-by: Sarah Monod --- tests/test_cdb2.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_cdb2.py b/tests/test_cdb2.py index 0fb7abe..3835705 100644 --- a/tests/test_cdb2.py +++ b/tests/test_cdb2.py @@ -38,7 +38,9 @@ def delete_all_rows(): def test_garbage_collecting_unused_handle(): - cdb2.Handle("mattdb", "dev").execute("select 1 union select 2") + cdb2.Handle("mattdb", "dev").execute( + "select /*CURSOR DRAIN TEST*/ 1 union select 2" + ) def test_commit_on_unused_connection(): From 7803b60523c91b268ad7ec4259b210a720558ea6 Mon Sep 17 00:00:00 2001 From: Sarah Monod Date: Thu, 6 Nov 2025 11:32:04 -0500 Subject: [PATCH 57/62] Release 1.9.1 Signed-off-by: Sarah Monod --- comdb2/_about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comdb2/_about.py b/comdb2/_about.py index d9c8367..fb3958c 100644 --- a/comdb2/_about.py +++ b/comdb2/_about.py @@ -1,2 +1,2 @@ __all__ = ["__version__"] -__version__ = "1.9.0" +__version__ = "1.9.1" From 6a86084fc9966869c9627a1c8c260c611de2cc78 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 31 Mar 2026 19:07:32 +0000 Subject: [PATCH 58/62] Bump actions/download-artifact from 6 to 7 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 6 to 7. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v6...v7) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ca0d069..4111434 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -74,7 +74,7 @@ jobs: needs: [build_bloomberg_comdb2] steps: - name: Download bloomberg-comdb2 with build artifacts - uses: actions/download-artifact@v6 + uses: actions/download-artifact@v8 with: name: bloomberg-comdb2 path: . @@ -118,7 +118,7 @@ jobs: - '3.8' steps: - name: Download comdb2 repo with build artifacts - uses: actions/download-artifact@v6 + uses: actions/download-artifact@v8 with: name: bloomberg-comdb2 path: . @@ -150,7 +150,7 @@ jobs: done ' - name: Download python-comdb2 sdist - uses: actions/download-artifact@v6 + uses: actions/download-artifact@v8 with: name: python-comdb2-sdist path: dist From 2be107137bcfbdc3e93369ef3e036397dc11e547 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 31 Mar 2026 19:28:54 +0000 Subject: [PATCH 59/62] Bump actions/upload-artifact from 5 to 6 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 5 to 6. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4111434..f4d188c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -63,7 +63,7 @@ jobs: - name: Archive bloomberg-comdb2 repo with build artifacts run: 'tar -C /opt -czvf bloomberg-comdb2.tar.gz bb' - name: Upload built bloomberg-comdb2 - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@v7 with: name: bloomberg-comdb2 path: ./bloomberg-comdb2.tar.gz @@ -97,7 +97,7 @@ jobs: pipx run build --sdist ' - name: Upload sdist - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@v7 with: name: python-comdb2-sdist path: dist/*.tar.gz From 454d92148edb8e233cc13d77d625dc79c56afc03 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 31 Mar 2026 19:31:00 +0000 Subject: [PATCH 60/62] Bump actions/checkout from 5 to 6 Bumps [actions/checkout](https://github.com/actions/checkout) from 5 to 6. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- .github/workflows/docs.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f4d188c..ecc582a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout bloomberg-comdb2 repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: repository: bloomberg/comdb2 path: bloomberg-comdb2 @@ -85,7 +85,7 @@ jobs: find /opt/bb && rm bloomberg-comdb2.tar.gz ' - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: 'Set up Python 3.8' uses: actions/setup-python@v6 # note that this step overwrites the PKG_CONFIG_PATH variable with: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c58a11f..2291008 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -20,7 +20,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: From ccc2555421072a83bbda9f69ca03c5e5b278ab87 Mon Sep 17 00:00:00 2001 From: Sarah Monod Date: Wed, 10 Jun 2026 11:03:56 -0400 Subject: [PATCH 61/62] Allow manual trigger of tests Signed-off-by: Sarah Monod --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ecc582a..2fb362f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,6 +1,7 @@ name: Tests on: + workflow_dispatch: push: pull_request: release: From c9f6431de415daf9df5d0982a62646cb9b5e5234 Mon Sep 17 00:00:00 2001 From: Sarah Monod Date: Wed, 10 Jun 2026 11:39:32 -0400 Subject: [PATCH 62/62] Add `apt-get update` before installation of pkgs This is necessary in order to ensure the packages can be installed in the first place. Signed-off-by: Sarah Monod --- .github/workflows/build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2fb362f..f70cc15 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -81,6 +81,7 @@ jobs: path: . - name: Install bloomberg-comdb2 run: ' + sudo apt-get update && sudo apt-get install -qy libprotobuf-c-dev libssl-dev libunwind-dev && sudo tar -C /opt -xzf bloomberg-comdb2.tar.gz && find /opt/bb && @@ -93,6 +94,7 @@ jobs: python-version: '3.8' # the lowest version that we support in CI - name: Build sdist run: ' + sudo apt-get update && sudo apt-get install -qy pkg-config && PKG_CONFIG_PATH=/opt/bb/lib/pkgconfig:/usr/lib/x86_64-linux-gnu/pkgconfig:$PKG_CONFIG_PATH pipx run build --sdist @@ -125,6 +127,7 @@ jobs: path: . - name: Install bloomberg-comdb2 run: ' + sudo apt-get update && sudo apt-get install -qy libprotobuf-c-dev libssl-dev libunwind-dev && sudo tar -C /opt -xzf bloomberg-comdb2.tar.gz && find /opt/bb && @@ -175,6 +178,7 @@ jobs: python-version: "${{matrix.python-version}}" - name: Install python-comdb2 from the sdist run: ' + sudo apt-get update && sudo apt-get install -qy pkg-config && python -m pip install --upgrade pip && PKG_CONFIG_PATH=/opt/bb/lib/pkgconfig:/usr/lib/x86_64-linux-gnu/pkgconfig:$PKG_CONFIG_PATH