diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..1230149 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 38a819b..f70cc15 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,6 +1,7 @@ name: Tests on: + workflow_dispatch: push: pull_request: release: @@ -24,7 +25,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout bloomberg-comdb2 repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: repository: bloomberg/comdb2 path: bloomberg-comdb2 @@ -46,6 +47,7 @@ jobs: ncurses-dev protobuf-c-compiler tcl + tcl-dev uuid-dev zlib1g-dev ' @@ -55,13 +57,14 @@ jobs: mkdir bloomberg-comdb2/build && cd bloomberg-comdb2/build && cmake .. && - make + make && + sudo make install ) ' - name: Archive bloomberg-comdb2 repo with build artifacts - run: 'tar czvf bloomberg-comdb2.tar.gz bloomberg-comdb2/' - - name: Upload bloomberg-comdb2 repo with build artifacts - uses: actions/upload-artifact@v3 + run: 'tar -C /opt -czvf bloomberg-comdb2.tar.gz bb' + - name: Upload built bloomberg-comdb2 + uses: actions/upload-artifact@v7 with: name: bloomberg-comdb2 path: ./bloomberg-comdb2.tar.gz @@ -72,36 +75,32 @@ jobs: needs: [build_bloomberg_comdb2] steps: - name: Download bloomberg-comdb2 with build artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v8 with: name: bloomberg-comdb2 path: . - name: Install bloomberg-comdb2 run: ' - sudo apt-get install -qy - libevent-dev - liblz4-dev - libprotobuf-c-dev - libsqlite3-dev - libssl-dev - libunwind-dev - zlib1g-dev && - tar xvf bloomberg-comdb2.tar.gz && - (cd bloomberg-comdb2/build && sudo make install) + sudo apt-get update && + sudo apt-get install -qy libprotobuf-c-dev libssl-dev libunwind-dev && + sudo tar -C /opt -xzf bloomberg-comdb2.tar.gz && + find /opt/bb && + rm bloomberg-comdb2.tar.gz ' - - uses: actions/checkout@v4 - - name: 'Set up Python 3.7' - uses: actions/setup-python@v4 # note that this step overwrites the PKG_CONFIG_PATH variable + - uses: actions/checkout@v6 + - name: 'Set up Python 3.8' + uses: actions/setup-python@v6 # note that this step overwrites the PKG_CONFIG_PATH variable with: - python-version: '3.7' # the lowest version that we support in CI + python-version: '3.8' # the lowest version that we support in CI - name: Build sdist run: ' + sudo apt-get update && sudo apt-get install -qy pkg-config && PKG_CONFIG_PATH=/opt/bb/lib/pkgconfig:/usr/lib/x86_64-linux-gnu/pkgconfig:$PKG_CONFIG_PATH pipx run build --sdist ' - name: Upload sdist - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v7 with: name: python-comdb2-sdist path: dist/*.tar.gz @@ -113,30 +112,26 @@ jobs: strategy: matrix: python-version: + - '3.14-dev' + - '3.13' - '3.12' - '3.11' - '3.10' - '3.9' - '3.8' - - '3.7' steps: - name: Download comdb2 repo with build artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v8 with: name: bloomberg-comdb2 path: . - name: Install bloomberg-comdb2 run: ' - sudo apt-get install -qy - libevent-dev - liblz4-dev - libprotobuf-c-dev - libsqlite3-dev - libssl-dev - libunwind-dev - zlib1g-dev && - tar xvf bloomberg-comdb2.tar.gz && - (cd bloomberg-comdb2/build && sudo make install) + sudo apt-get update && + sudo apt-get install -qy libprotobuf-c-dev libssl-dev libunwind-dev && + sudo tar -C /opt -xzf bloomberg-comdb2.tar.gz && + find /opt/bb && + rm bloomberg-comdb2.tar.gz ' - name: Start local comdb2 instance run: ' @@ -159,7 +154,7 @@ jobs: done ' - name: Download python-comdb2 sdist - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v8 with: name: python-comdb2-sdist path: dist @@ -177,12 +172,13 @@ jobs: echo "Creating $table_name from $table_file" /opt/bb/bin/cdb2sql "$COMDB2_DBNAME" local "create table $table_name { $(cat $table_file) }" done - - name: Set up Python ${{matrix.python_version}} - uses: actions/setup-python@v4 # note that this step overwrites the PKG_CONFIG_PATH variable + - name: Set up Python ${{matrix.python-version}} + uses: actions/setup-python@v6 # note that this step overwrites the PKG_CONFIG_PATH variable with: - python-version: "${{matrix.python_version}}" + python-version: "${{matrix.python-version}}" - name: Install python-comdb2 from the sdist run: ' + sudo apt-get update && sudo apt-get install -qy pkg-config && python -m pip install --upgrade pip && PKG_CONFIG_PATH=/opt/bb/lib/pkgconfig:/usr/lib/x86_64-linux-gnu/pkgconfig:$PKG_CONFIG_PATH diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..2291008 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,37 @@ +name: Docs + +on: + push: + pull_request: + release: + types: + - published + schedule: + # At 12:00 on every day-of-month + - cron: "0 12 */1 * *" + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + docs: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v6 + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Install Python dependencies + run: python3 -m pip install sphinx sphinx_rtd_theme + - name: Build docs + run: python -m sphinx docs html -a -E -n -v -W --keep-going + - name: Publish docs to GitHub Pages + if: github.event_name == 'release' && github.event.action == 'published' + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: html + single-commit: true diff --git a/.gitignore b/.gitignore index ba3a04f..e80a260 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /build +/comdb2.egg-info/ /dist /python_comdb2.egg-info *.pyc diff --git a/comdb2/__init__.py b/comdb2/__init__.py index da5f68f..c6382cc 100644 --- a/comdb2/__init__.py +++ b/comdb2/__init__.py @@ -12,7 +12,7 @@ """This package provides Python interfaces to Comdb2 databases. Two different Python submodules are provided for interacting with Comdb2 -databases. Both submodules work from Python 2.7+ and from Python 3.5+. +databases. `comdb2.dbapi2` provides an interface that conforms to `the Python Database API Specification v2.0 `_. If you're @@ -28,4 +28,6 @@ connections, this module may be simpler to get started with. """ -from ._about import * +from ._about import __version__ + +__all__ = ["__version__"] diff --git a/comdb2/_about.py b/comdb2/_about.py index 09033dd..fb3958c 100644 --- a/comdb2/_about.py +++ b/comdb2/_about.py @@ -1,2 +1,2 @@ -__all__ = ['__version__'] -__version__ = "1.4.1" +__all__ = ["__version__"] +__version__ = "1.9.1" diff --git a/comdb2/_ccdb2.pyx b/comdb2/_ccdb2.pyx index a439d91..e0de1ad 100644 --- a/comdb2/_ccdb2.pyx +++ b/comdb2/_ccdb2.pyx @@ -20,9 +20,7 @@ from cpython.ref cimport Py_TYPE, PyObject from cpython.mem cimport PyMem_Malloc, PyMem_Free import datetime - -from pytz import timezone, UTC -import six +import sys from ._cdb2_types import Error, Effects, DatetimeUs from . cimport _cdb2api as lib @@ -40,6 +38,11 @@ ctypedef fused client_datetime: lib.cdb2_client_datetimeus_t +cdef struct blob_descriptor: + size_t size + char* data + + cdef _string_as_bytes(s): if isinstance(s, unicode): return s.encode('utf-8') @@ -78,6 +81,11 @@ cdef _describe_exception(exc): cdef _bind_datetime(obj, client_datetime *val): + if obj.tzinfo is None: + obj = obj.replace(tzinfo=datetime.timezone.utc) + else: + obj = obj.astimezone(datetime.timezone.utc) + if client_datetime is lib.cdb2_client_datetimeus_t: val.usec = obj.microsecond else: @@ -100,20 +108,22 @@ cdef _bind_datetime(obj, client_datetime *val): cdef class _ParameterValue(object): - cdef int type + cdef lib.cdb2_coltype type cdef int size cdef void *data cdef object owner + cdef int list_size def __cinit__(self, obj, param_name): try: + self.list_size = -1 if obj is None: self.type = lib.CDB2_INTEGER self.owner = None self.size = 0 self.data = NULL return - elif isinstance(obj, six.integer_types): + elif isinstance(obj, int): self.type = lib.CDB2_INTEGER self.owner = None self.size = sizeof(long long) @@ -153,6 +163,62 @@ cdef class _ParameterValue(object): self.data = PyMem_Malloc(self.size) _bind_datetime(obj, self.data) return + elif isinstance(obj, (list, tuple)): + self.list_size = len(obj) + if 0 == self.list_size: + raise ValueError(f"empty {type(obj).__name__}s cannot be bound") + + if all(isinstance(ele, int) for ele in obj): + self.type = lib.CDB2_INTEGER + self.size = sizeof(long long) + self.owner = None + self.data = PyMem_Malloc(self.list_size * self.size) + for l_index in range(self.list_size): + (self.data)[l_index] = obj[l_index] + return + elif all(isinstance(ele, float) for ele in obj): + self.type = lib.CDB2_REAL + self.size = sizeof(double) + self.owner = None + self.data = PyMem_Malloc(self.list_size * self.size) + for l_index in range(self.list_size): + (self.data)[l_index] = obj[l_index] + return + elif all(isinstance(ele, bytes) for ele in obj): + if isinstance(obj, tuple): + owner = obj + else: + # Copy the list into a tuple. If we referenced the list + # directly, another thread could remove an element when + # we drop the GIL, invalidating a pointer we're holding + owner = tuple(obj) + + self.type = lib.CDB2_BLOB + self.size = sizeof(blob_descriptor) + self.owner = owner + self.data = PyMem_Malloc(self.list_size * self.size) + for l_index in range(self.list_size): + (self.data)[l_index].size = len(obj[l_index]) + (self.data)[l_index].data = obj[l_index] + return + elif all(isinstance(ele, unicode) for ele in obj): + self.type = lib.CDB2_CSTRING + self.size = sizeof(char*) + # Strings need to be converted to bytes + self.owner = [x.encode('utf-8') for x in obj] + self.data = PyMem_Malloc(self.list_size * self.size) + for l_index in range(self.list_size): + (self.data)[l_index] = (self.owner[l_index]) + return + elif not all(isinstance(ele, type(obj[0])) for ele in obj): + raise ValueError( + f"all {type(obj).__name__} elements must be the same type" + ) + else: + raise ValueError( + f"Cannot bind a {type(obj).__name__} of {type(obj[0]).__name__}" + ) + except Exception as e: exc = e else: @@ -161,15 +227,15 @@ cdef class _ParameterValue(object): exc_desc = _describe_exception(exc) if exc is not None: - errmsg = "Can't bind %s value %r for parameter '%s': %s" % ( + errmsg = "Can't bind %s value %r for parameter %r: %s" % ( type(obj).__name__, obj, param_name, exc_desc, ) - six.raise_from(Error(lib.CDB2ERR_CONV_FAIL, errmsg), exc) + raise Error(lib.CDB2ERR_CONV_FAIL, errmsg) from exc else: - errmsg = "Can't map %s value %r for parameter '%s' to a Comdb2 type" % ( + errmsg = "Can't map %s value %r for parameter %r to a Comdb2 type" % ( type(obj).__name__, obj, param_name, @@ -178,7 +244,7 @@ cdef class _ParameterValue(object): def __dealloc__(self): - if self.owner is None: + if self.owner is None or self.list_size != -1: PyMem_Free(self.data) @@ -199,11 +265,13 @@ cdef _make_datetime(client_datetime *val): val.tm.tm_min, val.tm.tm_sec, usec, - UTC, + datetime.timezone.utc, pytype) - return timezone(val.tzname).localize( - PyDateTimeAPI.DateTime_FromDateAndTime( + if sys.version_info >= (3, 14): + from zoneinfo import ZoneInfo + + return PyDateTimeAPI.DateTime_FromDateAndTimeAndFold( val.tm.tm_year + 1900, val.tm.tm_mon + 1, val.tm.tm_mday, @@ -211,9 +279,24 @@ cdef _make_datetime(client_datetime *val): val.tm.tm_min, val.tm.tm_sec, usec, - None, - pytype), - val.tm.tm_isdst) + ZoneInfo(val.tzname), + not val.tm.tm_isdst, + pytype) + else: + from pytz import timezone + + return timezone(val.tzname).localize( + PyDateTimeAPI.DateTime_FromDateAndTime( + val.tm.tm_year + 1900, + val.tm.tm_mon + 1, + val.tm.tm_mday, + val.tm.tm_hour, + val.tm.tm_min, + val.tm.tm_sec, + usec, + None, + pytype), + val.tm.tm_isdst) cdef _column_value(lib.cdb2_hndl_tp *hndl, int col): @@ -248,7 +331,7 @@ cdef _column_value(lib.cdb2_hndl_tp *hndl, int col): if exc is not None: errmsg += _describe_exception(exc) - six.raise_from(Error(lib.CDB2ERR_CONV_FAIL, errmsg), exc) + raise Error(lib.CDB2ERR_CONV_FAIL, errmsg) from exc else: errmsg += "Unsupported column type" raise Error(lib.CDB2ERR_NOTSUPPORTED, errmsg) @@ -289,7 +372,7 @@ cdef class _Cursor(object): ret = self.row_class(ret) except Exception as e: errmsg = "Instantiating row failed: " + _describe_exception(e) - six.raise_from(Error(lib.CDB2ERR_UNKNOWN, errmsg), e) + raise Error(lib.CDB2ERR_UNKNOWN, errmsg) from e return ret @@ -329,8 +412,8 @@ cdef class Handle(object): self.hndl = NULL _errchk(rc, self.hndl) - def execute(self, sql, parameters=None): - """execute(sql, params) -> Cursor over a SQL statement's result set""" + def execute(self, sql, parameters=None, int[::1] column_types=None): + """execute(sql, params, types) -> Cursor over a SQL statement's result set""" if not self.hndl: raise _closed_connection_error('execute') self.cursor = NULL sql = _string_as_bytes(sql) @@ -339,19 +422,46 @@ cdef class Handle(object): param_guards = [] try: if parameters is not None: - for key, val in parameters.items(): - ckey = _string_as_bytes(key) + if hasattr(parameters, "items"): + items = parameters.items() + bind_by_index = False + else: + items = enumerate(parameters, 1) + bind_by_index = True + + for key, val in items: + ckey = key if bind_by_index else _string_as_bytes(key) cval = _ParameterValue(val, key) param_guards.append(ckey) param_guards.append(cval) - rc = lib.cdb2_bind_param(self.hndl, ckey, - cval.type, cval.data, cval.size) + bind_array = (cval.list_size != -1) + if bind_array: + if bind_by_index: + rc = lib.cdb2_bind_array_index(self.hndl, ckey, + cval.type, cval.data, + cval.list_size, cval.size) + else: + rc = lib.cdb2_bind_array(self.hndl, ckey, + cval.type, cval.data, + cval.list_size, cval.size) + else: + if bind_by_index: + rc = lib.cdb2_bind_index(self.hndl, ckey, + cval.type, cval.data, cval.size) + else: + rc = lib.cdb2_bind_param(self.hndl, ckey, + cval.type, cval.data, cval.size) _errchk(rc, self.hndl) with nogil: while lib.cdb2_next_record(self.hndl) == lib.CDB2_OK: pass # consume any previous result set - rc = lib.cdb2_run_statement(self.hndl, c_sql) + if column_types is None: + rc = lib.cdb2_run_statement(self.hndl, c_sql) + else: + rc = lib.cdb2_run_statement_typed( + self.hndl, c_sql, len(column_types), &column_types[0] + ) _errchk(rc, self.hndl) finally: rc = lib.cdb2_clearbindings(self.hndl) @@ -393,7 +503,7 @@ cdef class Handle(object): row_class = self._row_factory(self.column_names()) except Exception as e: errmsg = "row_factory call failed: " + _describe_exception(e) - six.raise_from(Error(lib.CDB2ERR_UNKNOWN, errmsg), e) + raise Error(lib.CDB2ERR_UNKNOWN, errmsg) from e cdef _Cursor ret = _Cursor.__new__(_Cursor) ret.handle = self diff --git a/comdb2/_cdb2_types.py b/comdb2/_cdb2_types.py index 908c344..8b60d18 100644 --- a/comdb2/_cdb2_types.py +++ b/comdb2/_cdb2_types.py @@ -9,22 +9,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -from collections import namedtuple -from datetime import datetime -import six +from __future__ import annotations -__name__ = 'comdb2.cdb2' +import datetime +import enum +from typing import NamedTuple + +__name__ = "comdb2.cdb2" def _errstr(msg): try: - return msg.decode('utf-8') + return msg.decode("utf-8") except UnicodeDecodeError: # The DB's error strings aren't necessarily UTF-8. # If one isn't, it's preferable to mangle the error string than to # raise a UnicodeDecodeError (which would obscure the root cause). # Return a unicode string with \x escapes in place of non-ascii bytes. - return msg.decode('latin1').encode('unicode_escape').decode('ascii') + return msg.decode("latin1").encode("unicode_escape").decode("ascii") class Error(RuntimeError): @@ -35,31 +37,37 @@ class Error(RuntimeError): error_message (str): The string returned by cdb2api's ``cdb2_errstr`` after the failed call. """ - def __init__(self, error_code, error_message): - if not(isinstance(error_message, six.text_type)): + + def __init__(self, error_code: int, error_message: str) -> None: + if not (isinstance(error_message, str)): error_message = _errstr(error_message) self.error_code = error_code self.error_message = error_message - super(Error, self).__init__(error_code, error_message) + super().__init__(error_code, error_message) -class Effects(namedtuple('Effects', - "num_affected num_selected num_updated num_deleted num_inserted")): +class Effects(NamedTuple): """Type used to represent the count of rows affected by a SQL query. An object of this type is returned by `Handle.get_effects`. - - Attributes: - num_affected (int): The total number of rows that were affected. - num_selected (int): The number of rows that were selected. - num_updated (int): The number of rows that were updated. - num_deleted (int): The number of rows that were deleted. - num_inserted (int): The number of rows that were inserted. """ - __slots__ = () + num_affected: int + num_selected: int + num_updated: int + num_deleted: int + num_inserted: int -class DatetimeUs(datetime): + +# Override the auto-generated docstrings with more informative ones. +Effects.num_affected.__doc__ = "The total number of rows that were affected." +Effects.num_selected.__doc__ = "The number of rows that were selected." +Effects.num_updated.__doc__ = "The number of rows that were updated." +Effects.num_deleted.__doc__ = "The number of rows that were deleted." +Effects.num_inserted.__doc__ = "The number of rows that were inserted." + + +class DatetimeUs(datetime.datetime): """Provides a distinct representation for Comdb2's DATETIMEUS type. Historically, Comdb2 provided a DATETIME type with millisecond precision. @@ -81,49 +89,119 @@ class DatetimeUs(datetime): `.fromdatetime` alternate constructor, or any of the other alternate constructors inherited from `datetime.datetime`. """ + @classmethod - def fromdatetime(cls, dt): + def fromdatetime(cls, dt: datetime.datetime) -> DatetimeUs: """Return a `DatetimeUs` copied from a given `datetime.datetime`""" - fold = getattr(dt, 'fold', None) + fold = getattr(dt, "fold", None) kwargs = {} if fold is not None: - kwargs['fold'] = fold - - return DatetimeUs(dt.year, dt.month, dt.day, - dt.hour, dt.minute, dt.second, dt.microsecond, - dt.tzinfo, **kwargs) - - def __add__(self, other): - ret = super(DatetimeUs, self).__add__(other) - if isinstance(ret, datetime): + kwargs["fold"] = fold + + return DatetimeUs( + dt.year, + dt.month, + dt.day, + dt.hour, + dt.minute, + dt.second, + dt.microsecond, + dt.tzinfo, + **kwargs, + ) + + def __add__(self, other: datetime.timedelta) -> DatetimeUs: + ret = super().__add__(other) + if isinstance(ret, datetime.datetime): return DatetimeUs.fromdatetime(ret) return ret # must be a timedelta - def __sub__(self, other): - ret = super(DatetimeUs, self).__sub__(other) - if isinstance(ret, datetime): + def __sub__(self, other: datetime.timedelta) -> DatetimeUs: + ret = super().__sub__(other) + if isinstance(ret, datetime.datetime): return DatetimeUs.fromdatetime(ret) return ret # must be a timedelta - def __radd__(self, other): + def __radd__(self, other: datetime.timedelta) -> DatetimeUs: return self + other @classmethod - def now(cls, tz=None): - ret = super(DatetimeUs, cls).now(tz) + def now(cls, tz: datetime.tzinfo | None = None) -> DatetimeUs: + ret = super().now(tz) return DatetimeUs.fromdatetime(ret) @classmethod - def fromtimestamp(cls, timestamp, tz=None): - ret = super(DatetimeUs, cls).fromtimestamp(timestamp, tz) + def fromtimestamp( + cls, timestamp: float, tz: datetime.tzinfo | None = None + ) -> DatetimeUs: + ret = super().fromtimestamp(timestamp, tz) return DatetimeUs.fromdatetime(ret) def astimezone(self, *args, **kwargs): - ret = super(DatetimeUs, self).astimezone(*args, **kwargs) + ret = super().astimezone(*args, **kwargs) return DatetimeUs.fromdatetime(ret) def replace(self, *args, **kwargs): # Before Python 3.7, it is effectively implementation dependent whether # this returns a DatetimeUs or a datetime. - dt = super(DatetimeUs, self).replace(*args, **kwargs) + dt = super().replace(*args, **kwargs) return self.fromdatetime(dt) + + def as_datetime(self) -> datetime.datetime: + """Return a `datetime` representation.""" + return datetime.datetime( + year=self.year, + day=self.day, + month=self.month, + hour=self.hour, + minute=self.minute, + second=self.second, + microsecond=self.microsecond, + tzinfo=self.tzinfo, + fold=self.fold, + ) + + +class ColumnType(enum.IntEnum): + """This enum represents all known Comdb2 column types. + + Each value in the list returned by `Handle.column_types` will generally be + the value corresponding to one of the members of this enumeration, though + that's not always guaranteed because new types can be added to the Comdb2 + server at any time. + + A sequence of consisting of members of this enum can be passed as the + *column_types* parameter of `Handle.execute`. The database will coerce the + returned data to the given column types, or return an error if it cannot. + """ + + # The column types from cdb2_coltype in cdb2api.h, plus a few aliases. + INTEGER = 1 + REAL = 2 + CSTRING = 3 + BLOB = 4 + DATETIME = 6 + INTERVALYM = 7 + INTERVALDS = 8 + DATETIMEUS = 9 + INTERVALDSUS = 10 + INT = INTEGER + FLOAT = REAL + STRING = CSTRING + TEXT = CSTRING + BYTES = BLOB + + +class ConnectionFlags(enum.IntEnum): + """This enum represents connection flags. + + These values can be passed to the `Handle` constructor, either individually + or as a bitwise OR of multiple flags. + """ + + # The flags from cdb2_hndl_alloc_flags in cdb2api.h + READ_INTRANS_RESULTS = 2 + DIRECT_CPU = 4 + RANDOM = 8 + RANDOMROOM = 16 + ROOM = 32 diff --git a/comdb2/_cdb2api.pxd b/comdb2/_cdb2api.pxd index 40c84ab..2dfe554 100644 --- a/comdb2/_cdb2api.pxd +++ b/comdb2/_cdb2api.pxd @@ -18,7 +18,7 @@ cdef extern from "cdb2api.h" nogil: CDB2ERR_NOTSUPPORTED CDB2ERR_CONV_FAIL - enum: + enum cdb2_coltype: CDB2_INTEGER CDB2_REAL CDB2_CSTRING @@ -63,17 +63,21 @@ cdef extern from "cdb2api.h" nogil: ctypedef struct cdb2_hndl_tp: pass - int cdb2_open(cdb2_hndl_tp **hndl, const char *dbname, const char *type, int flags); - int cdb2_next_record(cdb2_hndl_tp *hndl); - int cdb2_get_effects(cdb2_hndl_tp *hndl, cdb2_effects_tp *effects); - int cdb2_close(cdb2_hndl_tp* hndl); - int cdb2_run_statement(cdb2_hndl_tp *hndl, const char *sql); - int cdb2_numcolumns(cdb2_hndl_tp* hndl); - const char* cdb2_column_name(cdb2_hndl_tp* hndl, int col); - int cdb2_column_type(cdb2_hndl_tp* hndl, int col); - int cdb2_column_size(cdb2_hndl_tp* hndl, int col); - void* cdb2_column_value(cdb2_hndl_tp* hndl, int col); - const char* cdb2_errstr(cdb2_hndl_tp* hndl); - int cdb2_bind_param(cdb2_hndl_tp *hndl, const char *name, int type, const void *varaddr, int length); - int cdb2_clearbindings(cdb2_hndl_tp *hndl); - int cdb2_clear_ack(cdb2_hndl_tp *hndl); + int cdb2_open(cdb2_hndl_tp **hndl, const char *dbname, const char *type, int flags) except + + int cdb2_next_record(cdb2_hndl_tp *hndl) except + + int cdb2_get_effects(cdb2_hndl_tp *hndl, cdb2_effects_tp *effects) except + + int cdb2_close(cdb2_hndl_tp* hndl) except + + int cdb2_run_statement(cdb2_hndl_tp *hndl, const char *sql) except + + int cdb2_run_statement_typed(cdb2_hndl_tp *hndl, const char *sql, int num_types, int *types) except + + int cdb2_numcolumns(cdb2_hndl_tp* hndl) except + + const char* cdb2_column_name(cdb2_hndl_tp* hndl, int col) except + + int cdb2_column_type(cdb2_hndl_tp* hndl, int col) except + + int cdb2_column_size(cdb2_hndl_tp* hndl, int col) except + + void* cdb2_column_value(cdb2_hndl_tp* hndl, int col) except + + const char* cdb2_errstr(cdb2_hndl_tp* hndl) except + + int cdb2_bind_param(cdb2_hndl_tp *hndl, const char *name, int type, const void *varaddr, int length) except + + int cdb2_bind_index(cdb2_hndl_tp *hndl, int index, int type, const void *varaddr, int length) except + + int cdb2_bind_array(cdb2_hndl_tp *hndl, const char *name, cdb2_coltype, const void *varaddr, size_t count, size_t typelen) except + + int cdb2_bind_array_index(cdb2_hndl_tp *hndl, int index, cdb2_coltype, const void *varaddr, size_t count, size_t typelen) except + + int cdb2_clearbindings(cdb2_hndl_tp *hndl) except + + int cdb2_clear_ack(cdb2_hndl_tp *hndl) except + diff --git a/comdb2/cdb2.py b/comdb2/cdb2.py index 37a67dd..e64038a 100644 --- a/comdb2/cdb2.py +++ b/comdb2/cdb2.py @@ -77,9 +77,14 @@ ----------------- In real applications you'll often need to pass parameters into a SQL query. -This is done using parameter binding - in the query, placeholders are specified -using ``@name``, and a mapping of names to values is passed to `Handle.execute` -along with the query. For example: +This is done using parameter binding, either by name or by position. + +By Name +~~~~~~~ + +In the query, placeholders are specified using ``@name`` or ``:name``, and a +mapping of names to values is passed to `Handle.execute` along with the query. +For example: >>> query = "select 25 between @a and @b" >>> print(list(hndl.execute(query, {'a': 20, 'b': 42}))) @@ -88,16 +93,33 @@ >>> print(list(hndl.execute(query, params))) [[0]] -In this example, we run the query with two different sets of -parameters, producing different results. First, we execute -the query with ``@a`` bound to ``20`` and ``@b`` bound to ``42``. In this -case, because ``20 <= 25 <= 42``, the expression evaluates to true, and a ``1`` -is returned. +In this example, we run the query with two different sets of parameters, +producing different results. First, we execute the query with ``@a`` bound to +``20`` and ``@b`` bound to ``42``. In this case, because ``20 <= 25 <= 42``, +the expression evaluates to true, and a single row with a single column +set to ``1`` is returned. + +When we run the same query with ``@b`` bound to ``23``, a row with a column +set to ``0`` is returned instead, because ``20 <= 25 <= 23`` is false. + +Note: + In both of these examples we make use of the `list` constructor to turn + the iterable returned by `Handle.execute` into a list of result rows. + +By Position +~~~~~~~~~~~ + +You can bind parameters positionally rather than by name, by using ``?`` +for each placeholder and providing a list or tuple of parameter values. +For example: + + >>> query = "select 25 between ? and ?" + >>> print(list(hndl.execute(query, [20, 42]))) + [[1]] -When we run the same query with ``@b`` bound to ``23``, a ``0`` -is returned instead, because ``20 <= 25 <= 23`` is false. In both of these -examples we make use of the `list` constructor to turn the iterable returned -by `Handle.execute` into a list of result rows. +Here, we execute the query with the first ``?`` bound to 20 and the second +``?`` bound to 42, so a row with a single column set to ``1`` is returned +like in the previous example. Types ----- @@ -112,8 +134,8 @@ NULL ``None`` integer `int` real `float` -blob `six.binary_type` (aka `bytes` in Python 3, ``str`` in Python 2) -text `six.text_type` (aka `str` in Python 3, ``unicode`` in Python 2) +blob `bytes` +text `str` datetime `datetime.datetime` datetimeus `DatetimeUs` ============ ================================================================ @@ -127,50 +149,95 @@ problems for new users. Make sure to carefully read :ref:`String and Blob Types` on that page. """ -from __future__ import absolute_import, unicode_literals -from ._cdb2_types import Error, Effects, DatetimeUs +from __future__ import annotations + +import array +import datetime +from collections.abc import Callable, Iterator, Mapping, Sequence +from typing import Any, List, Tuple, Union +from ._cdb2_types import Error, Effects, DatetimeUs, ColumnType, ConnectionFlags from ._ccdb2 import Handle as CHandle -__all__ = ['Error', 'Handle', 'Effects', 'DatetimeUs', - 'ERROR_CODE', 'TYPE', 'HANDLE_FLAGS'] +__all__ = [ + "Error", + "Handle", + "Effects", + "DatetimeUs", + "ERROR_CODE", + "TYPE", + "HANDLE_FLAGS", + "ColumnType", + "ConnectionFlags", + "Value", + "ParameterValue", +] + +Value = Union[ + None, + int, + float, + bytes, + str, + datetime.datetime, + DatetimeUs, +] +ParameterValue = Union[ + None, + int, + float, + bytes, + str, + datetime.datetime, + DatetimeUs, + List[int], + List[float], + List[bytes], + List[str], + Tuple[int, ...], + Tuple[float, ...], + Tuple[bytes, ...], + Tuple[str, ...], +] +Row = Any # Pull all comdb2 error codes from cdb2api.h into our namespace -ERROR_CODE = {u'CONNECT_ERROR' : -1, - u'NOTCONNECTED' : -2, - u'PREPARE_ERROR' : -3, - u'IO_ERROR' : -4, - u'INTERNAL' : -5, - u'NOSTATEMENT' : -6, - u'BADCOLUMN' : -7, - u'BADSTATE' : -8, - u'ASYNCERR' : -9, - u'INVALID_ID' : -12, - u'RECORD_OUT_OF_RANGE' : -13, - u'REJECTED' : -15, - u'STOPPED' : -16, - u'BADREQ' : -17, - u'DBCREATE_FAILED' : -18, - u'THREADPOOL_INTERNAL' : -20, - u'READONLY' : -21, - u'NOMASTER' : -101, - u'UNTAGGED_DATABASE' : -102, - u'CONSTRAINTS' : -103, - u'DEADLOCK' : 203, - u'TRAN_IO_ERROR' : -105, - u'ACCESS' : -106, - u'TRAN_MODE_UNSUPPORTED' : -107, - u'VERIFY_ERROR' : 2, - u'FKEY_VIOLATION' : 3, - u'NULL_CONSTRAINT' : 4, - u'CONV_FAIL' : 113, - u'NONKLESS' : 114, - u'MALLOC' : 115, - u'NOTSUPPORTED' : 116, - u'DUPLICATE' : 299, - u'TZNAME_FAIL' : 401, - u'UNKNOWN' : 300, - } +ERROR_CODE = { + "CONNECT_ERROR": -1, + "NOTCONNECTED": -2, + "PREPARE_ERROR": -3, + "IO_ERROR": -4, + "INTERNAL": -5, + "NOSTATEMENT": -6, + "BADCOLUMN": -7, + "BADSTATE": -8, + "ASYNCERR": -9, + "INVALID_ID": -12, + "RECORD_OUT_OF_RANGE": -13, + "REJECTED": -15, + "STOPPED": -16, + "BADREQ": -17, + "DBCREATE_FAILED": -18, + "THREADPOOL_INTERNAL": -20, + "READONLY": -21, + "NOMASTER": -101, + "UNTAGGED_DATABASE": -102, + "CONSTRAINTS": -103, + "DEADLOCK": 203, + "TRAN_IO_ERROR": -105, + "ACCESS": -106, + "TRAN_MODE_UNSUPPORTED": -107, + "VERIFY_ERROR": 2, + "FKEY_VIOLATION": 3, + "NULL_CONSTRAINT": 4, + "CONV_FAIL": 113, + "NONKLESS": 114, + "MALLOC": 115, + "NOTSUPPORTED": 116, + "DUPLICATE": 299, + "TZNAME_FAIL": 401, + "UNKNOWN": 300, +} """This dict maps all known Comdb2 error names to their respective values. The value returned in `Error.error_code` will generally be the value @@ -179,39 +246,22 @@ time. """ -# Pull comdb2 column types from cdb2api.h into our namespace -TYPE = {u'INTEGER' : 1, - u'REAL' : 2, - u'CSTRING' : 3, - u'BLOB' : 4, - u'DATETIME' : 6, - u'INTERVALYM' : 7, - u'INTERVALDS' : 8, - u'DATETIMEUS' : 9, - u'INTERVALDSUS' : 10, - } +TYPE = {e.name: e.value for e in ColumnType} """This dict maps all known Comdb2 types to their enumeration value. -Each value in the list returned by `Handle.column_types` will generally be the -value corresponding to one of the keys in this dict, though that's not always -guaranteed because new types can be added to the Comdb2 server at any time. +It predates the `ColumnType` enum and is retained only for backwards +compatibility. The `ColumnType` enum should be preferred for new usage. """ -# Pull comdb2 handle flags from cdb2api.h into our namespace -HANDLE_FLAGS = {u'READ_INTRANS_RESULTS' : 2, - u'DIRECT_CPU' : 4, - u'RANDOM' : 8, - u'RANDOMROOM' : 16, - u'ROOM' : 32, - } -"""This dict maps all known Comdb2 flags to their enumeration value. - -These values can be passed directly to `Handle`, though values not in this dict -can be passed as well (such as the bitwise OR of two different flags). +HANDLE_FLAGS = {e.name: e.value for e in ConnectionFlags} +"""This dict maps connection flags to their enumeration value. + +It predates the `ConnectionFlags` enum and is retained only for backwards +compatibility. The `ConnectionFlags` enum should be preferred for new usage. """ -class Handle(object): +class Handle: """Represents a connection to a database. By default, the connection will be made to the cluster configured as the @@ -248,23 +298,31 @@ class Handle(object): use a machine-specific default. """ - def __init__(self, database_name, tier="default", flags=0, tz='UTC', - host=None): + def __init__( + self, + database_name: str | bytes, + tier: str | bytes = "default", + flags: int = 0, + tz: str = "UTC", + host: str | bytes | None = None, + ) -> None: if host is not None: if tier != "default": - raise Error(ERROR_CODE['NOTSUPPORTED'], - "Connecting to a host by name and to a " - "cluster by tier are mutually exclusive") + raise Error( + ERROR_CODE["NOTSUPPORTED"], + "Connecting to a host by name and to a " + "cluster by tier are mutually exclusive", + ) else: tier = host - flags |= HANDLE_FLAGS['DIRECT_CPU'] + flags |= HANDLE_FLAGS["DIRECT_CPU"] self._hndl = CHandle(database_name, tier, flags) if tz is not None: self._hndl.execute("set timezone %s" % tz, {}) self._cursor = iter([]) - def close(self, ack_current_event=True): + def close(self, ack_current_event: bool = True) -> None: """Gracefully close the Comdb2 connection. Once a `Handle` has been closed, no further operations may be performed @@ -295,7 +353,7 @@ def close(self, ack_current_event=True): self._hndl.close(ack_current_event=ack_current_event) @property - def row_factory(self): + def row_factory(self) -> Callable[[list[str]], Callable[[list[Value]], Row]]: """Factory used when constructing result rows. By default, or when set to ``None``, each row is returned as a `list` @@ -315,22 +373,46 @@ def row_factory(self): return self._hndl.row_factory @row_factory.setter - def row_factory(self, value): + def row_factory( + self, value: Callable[[list[str]], Callable[[list[Value]], Row]] + ) -> None: self._hndl.row_factory = value - def execute(self, sql, parameters=None): + def execute( + self, + sql: str | bytes, + parameters: Mapping[str, ParameterValue] | Sequence[ParameterValue] | None = None, + *, + column_types: Sequence[ColumnType] | None = None, + ) -> Handle: """Execute a database operation (query or command). The ``sql`` string may have placeholders for parameters to be passed. This should always be the preferred method of parameterizing the SQL query, as it prevents SQL injection vulnerabilities and is faster. - Placeholders for named parameters must be in Comdb2's native format, - ``@param_name``. + Placeholders for named parameters must be in one of Comdb2's native + formats, either ``@param_name`` or ``:param_name``. Alternatively, you + can use ``?`` for each placeholder to bind parameters positionally + instead of by name. + + If ``column_types`` is provided and non-empty, it must be a sequence of + members of the `ColumnType` enumeration. The database will coerce the + data in the Nth column of the result set to the Nth given column type. + An error will be raised if the number of elements in ``column_types`` + doesn't match the number of columns in the result set, or if one of the + elements is not a supported column type, or if coercion fails. If + ``column_types`` is empty or not provided, no coercion is performed. Args: sql (str): The SQL string to execute. - parameters (Mapping[str, Any]): An optional mapping from parameter - names to the values to be bound for them. + parameters (Mapping[str, Any] | Sequence[Any]): + If the SQL statement has ``@param_name`` or ``:param_name`` + style placeholders, you must pass a mapping from parameter name + to value. If the SQL statement has ``?`` style placeholders, + you must instead pass an ordered sequence of parameter values. + column_types (Sequence[int]): An optional sequence of types (values + of the `ColumnType` enumeration) which the columns of the + result set will be coerced to. Returns: Handle: This method returns the `Handle` that it was called on, @@ -344,17 +426,29 @@ def execute(self, sql, parameters=None): Example: >>> for row in hndl.execute("select 1, 2 UNION ALL select @x, @y", - ... {'x': 2, 'y': 4}): + ... {"x": 2, "y": 4}): + ... print(row) + [1, 2] + [2, 4] + + >>> for row in hndl.execute("select 1, 2 UNION ALL select ?, ?", [2, 4]): ... print(row) [1, 2] [2, 4] """ if parameters is None: parameters = {} - self._cursor = iter(self._hndl.execute(sql, parameters)) + + column_types_array: Sequence[int] | None = None + if column_types is not None: + column_types_array = array.array("i", column_types) + if not column_types_array: + column_types_array = None + + self._cursor = iter(self._hndl.execute(sql, parameters, column_types_array)) return self - def __iter__(self): + def __iter__(self) -> Iterator[Row]: """Iterate over all remaining rows in the current result set. By default each row is returned as a `list`, where the elements in the @@ -375,9 +469,10 @@ def __next__(self): # compatibility for a user who did something like: # next(handle.execute("select 1")) return next(self._cursor) + next = __next__ - def get_effects(self): + def get_effects(self) -> Effects: """Return counts of rows affected by executed queries. Within a transaction, these counts are a running total from the start @@ -400,7 +495,7 @@ def get_effects(self): """ return self._hndl.get_effects() - def column_names(self): + def column_names(self) -> list[str]: """Returns the names of the columns of the current result set. Returns: @@ -408,7 +503,7 @@ def column_names(self): """ return self._hndl.column_names() - def column_types(self): + def column_types(self) -> list[int]: """Returns the type codes of the columns of the current result set. Returns: diff --git a/comdb2/cdb2.pyi b/comdb2/cdb2.pyi deleted file mode 100644 index c228b01..0000000 --- a/comdb2/cdb2.pyi +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright 2021 Bloomberg Finance L.P. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys - -from datetime import datetime -from datetime import timedelta -from datetime import tzinfo -from typing import Any -from typing import Callable -from typing import Dict -from typing import Iterator -from typing import List -from typing import Mapping -from typing import NamedTuple -from typing import Optional -from typing import Text -from typing import Union -from typing import overload - - -class Error(RuntimeError): - error_code: int - error_message: str - - def __init__(self, error_code: int, error_message: str) -> None: ... - - -class Effects(NamedTuple): - num_affected: int - num_selected: int - num_updated: int - num_deleted: int - num_inserted: int - - -class DatetimeUs(datetime): - @classmethod - def fromdatetime(cls, dt: datetime) -> DatetimeUs: ... - - def __add__(self, other: timedelta) -> DatetimeUs: ... - - # XXX: datetime derives from date, which provides a __sub__ for a date rhs. - # mypy doesn't like that datetime and its subclasses don't provide it. - @overload # type: ignore[override] - def __sub__( - self, - other: datetime, - ) -> timedelta: ... - - @overload - def __sub__(self, other: timedelta) -> DatetimeUs: ... - - def __radd__(self, other: timedelta) -> DatetimeUs: ... - - @classmethod - def now(cls, tz: Optional[tzinfo] = ...) -> DatetimeUs: ... - - @classmethod - def fromtimestamp( - cls, timestamp: float, tz: Optional[tzinfo] = ... - ) -> DatetimeUs: ... - - if sys.version_info >= (3, 3): - def astimezone(self, tz: Optional[tzinfo]=...) -> DatetimeUs: ... - else: - def astimezone(self, tz: tzinfo) -> DatetimeUs: ... - - if sys.version_info >= (3, 6): - def replace( - self, - year: int = ..., - month: int = ..., - day: int = ..., - hour: int = ..., - minute: int = ..., - second: int = ..., - microsecond: int = ..., - tzinfo: Optional[tzinfo] = ..., - *, - fold: int = ..., - ) -> DatetimeUs: ... - else: - def replace( - self, - year: int = ..., - month: int = ..., - day: int = ..., - hour: int = ..., - minute: int = ..., - second: int = ..., - microsecond: int = ..., - tzinfo: Optional[tzinfo] = ..., - ) -> DatetimeUs: ... - - -ERROR_CODE: Dict[Text, int] -TYPE: Dict[Text, int] -HANDLE_FLAGS: Dict[Text, int] - -Value = Union[None, int, float, bytes, Text, datetime, DatetimeUs] -Row = Any - - -class Handle: - def __init__( - self, - database_name: Union[Text, bytes], - tier: Union[Text, bytes] = ..., - flags: int = ..., - tz: str = ..., - host: Optional[Union[Text, bytes]] = ..., - ) -> None: ... - - def close(self, *, ack_current_event=...) -> None: ... - - @property - def row_factory(self) -> Callable[[List[Text]], Callable[[List[Value]], Row]]: ... - - @row_factory.setter - def row_factory( - self, value: Callable[[List[Text]], Callable[[List[Value]], Row]] - ) -> None: ... - - def execute( - self, - sql: Union[Text, bytes], - parameters: Optional[Mapping[Text, Value]] = ..., - ) -> Handle: ... - - def __iter__(self) -> Iterator[Row]: ... - - def get_effects(self) -> Effects: ... - - def column_names(self) -> List[Text]: ... - - def column_types(self) -> List[int]: ... diff --git a/comdb2/dbapi2.py b/comdb2/dbapi2.py index cacf68c..26a057f 100644 --- a/comdb2/dbapi2.py +++ b/comdb2/dbapi2.py @@ -99,10 +99,15 @@ ----------------- In real applications you'll often need to pass parameters into a SQL query. -This is done using parameter binding - in the query, placeholders are specified -using ``%(name)s``, and a mapping of ``name`` to parameter value is passed to -`Cursor.execute` along with the query. The ``%(`` and ``)s`` are fixed, and -the ``name`` between them varies for each parameter. For example: +This is done using parameter binding, either by name or by position. + +By Name +~~~~~~~ + +In the query, placeholders are specified using ``%(name)s``, and a mapping of +``name`` to parameter value is passed to `Cursor.execute` along with the query. +The ``%(`` and ``)s`` are fixed, and the ``name`` inside them varies for each +parameter. For example: >>> query = "select 25 between %(a)s and %(b)s" >>> print(conn.cursor().execute(query, {'a': 20, 'b': 42}).fetchall()) @@ -114,17 +119,59 @@ In this example, we run the query with two different sets of parameters, producing different results. First, we execute the query with parameter ``a`` bound to ``20`` and ``b`` bound to ``42``. In this case, -because ``20 <= 25 <= 42``, the expression evaluates to true, and a ``1`` is -returned. +because ``20 <= 25 <= 42``, the expression evaluates to true, and a row +containing a single column with a value of ``1`` is returned. -When we run the same query with parameter ``b`` bound to ``23``, a ``0`` is -returned instead, because ``20 <= 25 <= 23`` is false. +When we run the same query with parameter ``b`` bound to ``23``, a row +containing a single column with a value of ``0`` is returned instead, because +``20 <= 25 <= 23`` is false. -Note: - Because parameters are bound using ``%(name)s``, other ``%`` signs in +Warning: + Because named parameters are bound using ``%(name)s``, other ``%`` signs in a query must be escaped. For example, ``WHERE name like 'M%'`` becomes ``WHERE name LIKE 'M%%'``. +Danger: + This applies even when no parameters are passed at all - passing ``None`` + or no parameters behaves the same as passing an empty `dict`. + +By Position +~~~~~~~~~~~ + +You can bind parameters positionally rather than by name, by using ``?`` +for each placeholder and providing a list or tuple of parameter values. +For example: + + >>> query = "select 25 between ? and ?" + >>> print(conn.cursor().execute(query, [20, 42]).fetchall()) + [[1]] + +In this example, we execute the query with the first ``?`` bound to 20 and the +second ``?`` bound to 42, so a row with a single column with a value of ``1`` +is returned like in the previous example. + +Warning: + Unlike when binding parameters by name, you must not escape ``%`` signs in + the SQL when binding parameters positionally. + + For example, you could do ``WHERE val % 5 = ?``, but not ``WHERE val %% 5 = ?``. + +Tip: + You can pass an empty tuple of parameters to avoid the need to escape ``%`` + signs in the SQL even when you don't want to bind any parameters, like: + + >>> query = "select 42 % 20" + >>> print(conn.cursor().execute(query, ()).fetchall()) + [[2]] + + Compare this against what happens if you don't pass any parameters at all: + + >>> print(conn.cursor().execute(query).fetchall()) + Traceback (most recent call last): + ... + comdb2.dbapi2.InterfaceError: Invalid Python format string for query + + Types ----- @@ -138,8 +185,8 @@ NULL ``None`` integer `int` real `float` -blob `six.binary_type` (aka `bytes` in Python 3, ``str`` in Python 2) -text `six.text_type` (aka `str` in Python 3, ``unicode`` in Python 2) +blob `bytes` +text `str` datetime `datetime.datetime` datetimeus `DatetimeUs` ============ ================================================================ @@ -239,27 +286,57 @@ `Connection.cursor` is called, we call `Cursor.close` on any existing, open cursor for that connection. """ -from __future__ import absolute_import, unicode_literals + +from __future__ import annotations import functools import itertools import weakref import datetime import re -import six from . import cdb2 - -__all__ = ['apilevel', 'threadsafety', 'paramstyle', - 'connect', 'Connection', 'Cursor', - 'STRING', 'BINARY', 'NUMBER', 'DATETIME', 'ROWID', - 'Datetime', 'DatetimeUs', 'Binary', 'Timestamp', 'TimestampUs', - 'DatetimeFromTicks', 'DatetimeUsFromTicks', 'TimestampFromTicks', - 'Error', 'Warning', 'InterfaceError', 'DatabaseError', - 'InternalError', 'OperationalError', 'ProgrammingError', - 'IntegrityError', 'DataError', 'NotSupportedError', - 'UniqueKeyConstraintError', 'ForeignKeyConstraintError', - 'NonNullConstraintError'] +from .cdb2 import ColumnType, Row, Value, ParameterValue +from collections.abc import Callable, Iterator, Mapping, Sequence +from typing import Any, List + +__all__ = [ + "apilevel", + "threadsafety", + "paramstyle", + "connect", + "ColumnType", + "Connection", + "Cursor", + "STRING", + "BINARY", + "NUMBER", + "DATETIME", + "ROWID", + "Datetime", + "DatetimeUs", + "Binary", + "Timestamp", + "TimestampUs", + "DatetimeFromTicks", + "DatetimeUsFromTicks", + "TimestampFromTicks", + "Error", + "Warning", + "InterfaceError", + "DatabaseError", + "InternalError", + "OperationalError", + "ProgrammingError", + "IntegrityError", + "DataError", + "NotSupportedError", + "UniqueKeyConstraintError", + "ForeignKeyConstraintError", + "NonNullConstraintError", + "Value", + "ParameterValue", +] apilevel = "2.0" """This module conforms to the Python Database API Specification 2.0.""" @@ -273,17 +350,31 @@ Comdb2's native placeholder format is ``@name``, but that cannot be used by this module because it's not an acceptable `DB-API 2.0 placeholder style `_. This module uses -``pyformat`` because it is the only DB-API 2.0 paramstyle that we can translate -into Comdb2's placeholder format without needing a SQL parser. +``pyformat`` for named parameters because it is the only DB-API 2.0 paramstyle +that we can translate into Comdb2's placeholder format without needing +a SQL parser. This module also supports the ``qmark`` parameter style for +binding parameters positionally. Note: An int value is bound as ``%(my_int)s``, not as ``%(my_int)d`` - the last character is always ``s``. Note: - Because SQL strings for this module use the ``pyformat`` placeholder style, - any literal ``%`` characters in a query must be escaped by doubling them. - ``WHERE name like 'M%'`` becomes ``WHERE name LIKE 'M%%'``. + When binding parameters by name, any ``%`` sign is recognized as the start + of a ``pyformat`` style placeholder, and so any literal ``%`` characters in + a SQL statement must be escaped by doubling. ``WHERE name like 'M%'`` + becomes ``WHERE name LIKE 'M%%'``. This does not apply when binding + parameters positionally with ``?`` placeholders, nor when the literal ``%`` + appears in a parameter value as opposed to literally in the query. + In either of those cases, the ``%`` characters must not be escaped. + +Warning: + Literal ``%`` signs in the query must be escaped when no parameters are + passed at all -- passing ``None`` or no parameters behaves the same as + passing an empty `dict`. You can avoid the need to escape ``%`` signs in + an unparametrized query by instead passing an empty `tuple` as parameters, + which causes the statement to be treated as having ``qmark`` placeholders + instead of ``pyformat`` placeholders. """ _FIRST_WORD_OF_STMT = re.compile( @@ -298,13 +389,13 @@ \s* # then skip over any whitespace (\w+) # and capture the first word """, - re.VERBOSE | re.DOTALL | (0 if six.PY2 else re.ASCII), + re.VERBOSE | re.DOTALL | re.ASCII, ) -_VALID_SP_NAME = re.compile(r'^[A-Za-z0-9_.]+$') +_VALID_SP_NAME = re.compile(r"^[A-Za-z0-9_.]+$") @functools.total_ordering -class _TypeObject(object): +class _TypeObject: def __init__(self, *value_names): self.value_names = value_names self.values = [cdb2.TYPE[v] for v in value_names] @@ -316,24 +407,25 @@ def __lt__(self, other): return self != other and other < self.values def __repr__(self): - return 'TypeObject' + str(self.value_names) + return "TypeObject" + str(self.value_names) -def _binary(string): - if isinstance(string, six.text_type): - return string.encode('utf-8') +def _binary(string: str | bytes) -> bytes: + if isinstance(string, str): + return string.encode("utf-8") return bytes(string) -STRING = _TypeObject('CSTRING') + +STRING = _TypeObject("CSTRING") """The type codes of TEXT result columns compare equal to this constant.""" -BINARY = _TypeObject('BLOB') +BINARY = _TypeObject("BLOB") """The type codes of BLOB result columns compare equal to this constant.""" -NUMBER = _TypeObject('INTEGER', 'REAL') +NUMBER = _TypeObject("INTEGER", "REAL") """The type codes of numeric result columns compare equal to this constant.""" -DATETIME = _TypeObject('DATETIME', 'DATETIMEUS') +DATETIME = _TypeObject("DATETIME", "DATETIMEUS") """The type codes of datetime result columns compare equal to this constant.""" ROWID = STRING @@ -350,10 +442,7 @@ def _binary(string): TimestampFromTicks = Timestamp.fromtimestamp TimestampUsFromTicks = TimestampUs.fromtimestamp -try: - UserException = StandardError # Python 2 -except NameError: - UserException = Exception # Python 3 +UserException = Exception class Error(UserException): @@ -364,6 +453,7 @@ class Error(UserException): `Connection` objects, to simplify error handling in environments where multiple connections from different modules are used. """ + pass @@ -372,21 +462,25 @@ class Warning(UserException): This is required to exist by the DB-API interface, but we never raise it. """ + pass class InterfaceError(Error): """Exception raised for errors caused by misuse of this module.""" + pass class DatabaseError(Error): """Base class for all errors reported by the database.""" + pass class InternalError(DatabaseError): """Exception raised for internal errors reported by the database.""" + pass @@ -396,6 +490,7 @@ class OperationalError(DatabaseError): These errors are not necessarily the result of a bug either in the application or in the database - for example, dropped connections. """ + pass @@ -405,6 +500,7 @@ class ProgrammingError(DatabaseError): For example, this will be raised for syntactically incorrect SQL, or for passing a different number of parameters than are required by the query. """ + pass @@ -416,6 +512,7 @@ class IntegrityError(DatabaseError): an index may not have duplicates. Other types of constraint violations may raise this type directly. """ + pass @@ -428,6 +525,7 @@ class UniqueKeyConstraintError(IntegrityError): .. versionadded:: 1.1 """ + pass @@ -444,6 +542,7 @@ class ForeignKeyConstraintError(IntegrityError): .. versionadded:: 1.1 """ + pass @@ -456,6 +555,7 @@ class NonNullConstraintError(IntegrityError): .. versionadded:: 1.1 """ + pass @@ -466,68 +566,60 @@ class DataError(DatabaseError): of range for the column type that would store it, or if you specify an invalid timezone name for the connection. """ + pass class NotSupportedError(DatabaseError): """Exception raised when unsupported operations are attempted.""" + pass _EXCEPTION_BY_RC = { - cdb2.ERROR_CODE['CONNECT_ERROR'] : OperationalError, - cdb2.ERROR_CODE['NOTCONNECTED'] : ProgrammingError, - cdb2.ERROR_CODE['PREPARE_ERROR'] : ProgrammingError, - cdb2.ERROR_CODE['IO_ERROR'] : OperationalError, - cdb2.ERROR_CODE['INTERNAL'] : InternalError, - cdb2.ERROR_CODE['NOSTATEMENT'] : ProgrammingError, - cdb2.ERROR_CODE['BADCOLUMN'] : ProgrammingError, - cdb2.ERROR_CODE['BADSTATE'] : ProgrammingError, - cdb2.ERROR_CODE['ASYNCERR'] : OperationalError, - - cdb2.ERROR_CODE['INVALID_ID'] : InternalError, - cdb2.ERROR_CODE['RECORD_OUT_OF_RANGE'] : OperationalError, - - cdb2.ERROR_CODE['REJECTED'] : OperationalError, - cdb2.ERROR_CODE['STOPPED'] : OperationalError, - cdb2.ERROR_CODE['BADREQ'] : OperationalError, - cdb2.ERROR_CODE['DBCREATE_FAILED'] : OperationalError, - - cdb2.ERROR_CODE['THREADPOOL_INTERNAL'] : OperationalError, - cdb2.ERROR_CODE['READONLY'] : NotSupportedError, - - cdb2.ERROR_CODE['NOMASTER'] : InternalError, - cdb2.ERROR_CODE['UNTAGGED_DATABASE'] : NotSupportedError, - cdb2.ERROR_CODE['CONSTRAINTS'] : IntegrityError, - cdb2.ERROR_CODE['DEADLOCK'] : OperationalError, - - cdb2.ERROR_CODE['TRAN_IO_ERROR'] : OperationalError, - cdb2.ERROR_CODE['ACCESS'] : OperationalError, - - cdb2.ERROR_CODE['TRAN_MODE_UNSUPPORTED'] : NotSupportedError, - - cdb2.ERROR_CODE['VERIFY_ERROR'] : OperationalError, - cdb2.ERROR_CODE['FKEY_VIOLATION'] : ForeignKeyConstraintError, - cdb2.ERROR_CODE['NULL_CONSTRAINT'] : NonNullConstraintError, - - cdb2.ERROR_CODE['CONV_FAIL'] : DataError, - cdb2.ERROR_CODE['NONKLESS'] : NotSupportedError, - cdb2.ERROR_CODE['MALLOC'] : OperationalError, - cdb2.ERROR_CODE['NOTSUPPORTED'] : NotSupportedError, - - cdb2.ERROR_CODE['DUPLICATE'] : UniqueKeyConstraintError, - cdb2.ERROR_CODE['TZNAME_FAIL'] : DataError, - - cdb2.ERROR_CODE['UNKNOWN'] : OperationalError, + cdb2.ERROR_CODE["CONNECT_ERROR"]: OperationalError, + cdb2.ERROR_CODE["NOTCONNECTED"]: ProgrammingError, + cdb2.ERROR_CODE["PREPARE_ERROR"]: ProgrammingError, + cdb2.ERROR_CODE["IO_ERROR"]: OperationalError, + cdb2.ERROR_CODE["INTERNAL"]: InternalError, + cdb2.ERROR_CODE["NOSTATEMENT"]: ProgrammingError, + cdb2.ERROR_CODE["BADCOLUMN"]: ProgrammingError, + cdb2.ERROR_CODE["BADSTATE"]: ProgrammingError, + cdb2.ERROR_CODE["ASYNCERR"]: OperationalError, + cdb2.ERROR_CODE["INVALID_ID"]: InternalError, + cdb2.ERROR_CODE["RECORD_OUT_OF_RANGE"]: OperationalError, + cdb2.ERROR_CODE["REJECTED"]: OperationalError, + cdb2.ERROR_CODE["STOPPED"]: OperationalError, + cdb2.ERROR_CODE["BADREQ"]: OperationalError, + cdb2.ERROR_CODE["DBCREATE_FAILED"]: OperationalError, + cdb2.ERROR_CODE["THREADPOOL_INTERNAL"]: OperationalError, + cdb2.ERROR_CODE["READONLY"]: NotSupportedError, + cdb2.ERROR_CODE["NOMASTER"]: InternalError, + cdb2.ERROR_CODE["UNTAGGED_DATABASE"]: NotSupportedError, + cdb2.ERROR_CODE["CONSTRAINTS"]: IntegrityError, + cdb2.ERROR_CODE["DEADLOCK"]: OperationalError, + cdb2.ERROR_CODE["TRAN_IO_ERROR"]: OperationalError, + cdb2.ERROR_CODE["ACCESS"]: OperationalError, + cdb2.ERROR_CODE["TRAN_MODE_UNSUPPORTED"]: NotSupportedError, + cdb2.ERROR_CODE["VERIFY_ERROR"]: OperationalError, + cdb2.ERROR_CODE["FKEY_VIOLATION"]: ForeignKeyConstraintError, + cdb2.ERROR_CODE["NULL_CONSTRAINT"]: NonNullConstraintError, + cdb2.ERROR_CODE["CONV_FAIL"]: DataError, + cdb2.ERROR_CODE["NONKLESS"]: NotSupportedError, + cdb2.ERROR_CODE["MALLOC"]: OperationalError, + cdb2.ERROR_CODE["NOTSUPPORTED"]: NotSupportedError, + cdb2.ERROR_CODE["DUPLICATE"]: UniqueKeyConstraintError, + cdb2.ERROR_CODE["TZNAME_FAIL"]: DataError, + cdb2.ERROR_CODE["UNKNOWN"]: OperationalError, } def _raise_wrapped_exception(exc): code = exc.error_code - msg = '%s (cdb2api rc %d)' % (exc.error_message, code) + msg = "%s (cdb2api rc %d)" % (exc.error_message, code) if "null constraint violation" in msg: - six.raise_from(NonNullConstraintError(msg), exc) # DRQS 86013831 - six.raise_from(_EXCEPTION_BY_RC.get(code, OperationalError)(msg), exc) + raise NonNullConstraintError(msg) from exc # DRQS 86013831 + raise _EXCEPTION_BY_RC.get(code, OperationalError)(msg) from exc def _sql_operation(sql): @@ -538,7 +630,7 @@ def _sql_operation(sql): def _operation_ends_transaction(operation): - return operation == 'commit' or operation == 'rollback' + return operation == "commit" or operation == "rollback" def _modifies_rows(operation): @@ -546,10 +638,15 @@ def _modifies_rows(operation): # exec is deliberately excluded because it might return a result set, and # this function is used to determine whether it's safe to call # cdb2_get_effects after running the operation. - return operation in ('commit', 'insert', 'update', 'delete') + return operation in ("commit", "insert", "update", "delete") -def connect(*args, **kwargs): +def connect( + database_name: str | bytes, + tier: str | bytes = "default", + autocommit: bool = False, + host: str | bytes | None = None, +) -> Connection: """Establish a connection to a Comdb2 database. All arguments are passed directly through to the `Connection` constructor. @@ -563,10 +660,15 @@ def connect(*args, **kwargs): Returns: Connection: A handle for the newly established connection. """ - return Connection(*args, **kwargs) + return Connection( + database_name=database_name, + tier=tier, + autocommit=autocommit, + host=host, + ) -class Connection(object): +class Connection: """Represents a connection to a Comdb2 database. By default, the connection will be made to the cluster configured as the @@ -624,11 +726,18 @@ class Connection(object): statements, disabling DB-API 2.0's automatic implicit transactions. """ - def __init__(self, database_name, tier="default", autocommit=False, - host=None): + def __init__( + self, + database_name: str | bytes, + tier: str | bytes = "default", + autocommit: bool = False, + host: str | bytes | None = None, + ) -> None: if host is not None and tier != "default": - raise InterfaceError("Connecting to a host by name and to a " - "cluster by tier are mutually exclusive") + raise InterfaceError( + "Connecting to a host by name and to a " + "cluster by tier are mutually exclusive" + ) self._active_cursor = None self._in_transaction = False @@ -643,7 +752,7 @@ def _check_closed(self): raise InterfaceError("Attempted to use a closed Connection") @property - def row_factory(self): + def row_factory(self) -> Callable[[list[str]], Callable[[list[Value]], Row]]: """Factory used when constructing result rows. By default, or when set to ``None``, each row is returned as a `list` @@ -665,7 +774,9 @@ def row_factory(self): return self._hndl.row_factory @row_factory.setter - def row_factory(self, value): + def row_factory( + self, value: Callable[[list[str]], Callable[[list[Value]], Row]] + ) -> None: self._check_closed() self._hndl.row_factory = value @@ -683,7 +794,7 @@ def _execute(self, operation): cursor = self.cursor() cursor._execute(operation, operation) - def close(self, ack_current_event=True): + def close(self, ack_current_event: bool = True) -> None: """Gracefully close the Comdb2 connection. Once a `Connection` has been closed, no further operations may be @@ -716,7 +827,7 @@ def close(self, ack_current_event=True): self._hndl.close(ack_current_event=ack_current_event) self._hndl = None - def commit(self): + def commit(self) -> None: """Commit any pending transaction to the database. This method will fail if the `Connection` is in ``autocommit`` mode and @@ -725,7 +836,7 @@ def commit(self): self._check_closed() self._execute("commit") - def rollback(self): + def rollback(self) -> None: """Rollback the current transaction. This method will fail if the `Connection` is in ``autocommit`` mode and @@ -741,7 +852,7 @@ def rollback(self): self._check_closed() self._execute("rollback") - def cursor(self): + def cursor(self) -> Cursor: """Return a new `Cursor` for this connection. This calls `Cursor.close` on any outstanding `Cursor`; only one @@ -774,7 +885,7 @@ def cursor(self): NotSupportedError = NotSupportedError -class Cursor(object): +class Cursor: """Class used to send requests through a database connection. This class is not meant to be instantiated directly; it should always be @@ -792,12 +903,12 @@ class Cursor(object): """ _ErrorMessagesByOperation = { - 'begin': "Transactions may not be started explicitly", - 'commit': "Use Connection.commit to commit transactions", - 'rollback': "Use Connection.rollback to roll back transactions", + "begin": "Transactions may not be started explicitly", + "commit": "Use Connection.commit to commit transactions", + "rollback": "Use Connection.rollback to roll back transactions", } - def __init__(self, conn): + def __init__(self, conn: Connection) -> None: self._arraysize = 1 self._conn = conn self._hndl = conn._hndl @@ -810,7 +921,7 @@ def _check_closed(self): raise InterfaceError("Attempted to use a closed cursor") @property - def arraysize(self): + def arraysize(self) -> int: """Controls the number of rows to fetch at a time with `fetchmany`. The default is ``1``, meaning that a single row will be fetched at @@ -819,11 +930,13 @@ def arraysize(self): return self._arraysize @arraysize.setter - def arraysize(self, value): + def arraysize(self, value: int) -> None: self._arraysize = value @property - def description(self): + def description( + self, + ) -> tuple[tuple[str, object, None, None, None, None, None], ...] | None: """Provides the name and type of each column in the latest result set. This read-only attribute will contain one element per column in the @@ -858,7 +971,7 @@ def description(self): return self._description @property - def rowcount(self): + def rowcount(self) -> int: """Provides the count of rows modified by the last transaction. For `Cursor` objects on a `Connection` that is not using ``autocommit`` @@ -887,12 +1000,12 @@ def rowcount(self): # Optional DB API Extension @property - def connection(self): + def connection(self) -> Connection: """Return a reference to the `Connection` that this `Cursor` uses.""" self._check_closed() return self._conn - def close(self): + def close(self) -> None: """Close the cursor now. From this point forward an exception will be raised if any @@ -908,7 +1021,7 @@ def close(self): self._description = None self._closed = True - def callproc(self, procname, parameters): + def callproc(self, procname: str, parameters: Sequence[ParameterValue]) -> Sequence[ParameterValue]: """Call a stored procedure with the given name. The ``parameters`` sequence must contain one entry for each argument @@ -931,18 +1044,35 @@ def callproc(self, procname, parameters): if not _VALID_SP_NAME.match(procname): raise NotSupportedError("Invalid procedure name '%s'" % procname) params_as_dict = {str(i): e for i, e in enumerate(parameters)} - sql = ("exec procedure " + procname + "(" - + ", ".join("%%(%d)s" % i for i in range(len(params_as_dict))) - + ")") + sql = ( + "exec procedure " + + procname + + "(" + + ", ".join("%%(%d)s" % i for i in range(len(params_as_dict))) + + ")" + ) self.execute(sql, params_as_dict) return parameters[:] - def execute(self, sql, parameters=None): + def execute( + self, + sql: str, + parameters: Mapping[str, ParameterValue] | Sequence[ParameterValue] | None = None, + *, + column_types: Sequence[ColumnType] | None = None, + ) -> Cursor: """Execute a database operation (query or command). - The ``sql`` string must be provided as a Python format string, with - parameter placeholders represented as ``%(name)s`` and all other ``%`` - signs escaped as ``%%``. + The ``sql`` string may contain either named placeholders represented + as ``%(name)s`` or positionally ordered placeholders represented + as ``?``. + + When the parameters are a mapping or ``None``, named placeholders are + being used, and so any literal ``%`` signs in the statement must be + escaped by doubling them, to distinguish them from the start of a named + placeholder. When a sequence of parameters is provided instead, any + placeholders must be positional ``?`` placeholders, and literal ``%`` + signs in the SQL must not be escaped. Note: Using placeholders should always be the preferred method of @@ -950,10 +1080,29 @@ def execute(self, sql, parameters=None): vulnerabilities, and is faster than dynamically building SQL strings. + If ``column_types`` is provided and non-empty, it must be a sequence of + members of the `ColumnType` enumeration. The database will coerce the + data in the Nth column of the result set to the Nth given column type. + An error will be raised if the number of elements in ``column_types`` + doesn't match the number of columns in the result set, or if one of the + elements is not a supported column type, or if coercion fails. If + ``column_types`` is empty or not provided, no coercion is performed. + + Note: + Databases APIs are not required to allow result set column types to + be specified explicitly. We allow this as a non-standard DB-API 2.0 + extension. + Args: sql (str): The SQL string to execute, as a Python format string. - parameters (Mapping[str, Any]): An optional mapping from parameter - names to the values to be bound for them. + parameters (Mapping[str, Any] | Sequence[Any]): + If the SQL statement has ``%(param_name)s`` style placeholders, + you must pass a mapping from parameter name to value. + If the SQL statement has ``?`` style placeholders, you must + instead pass an ordered sequence of parameter values. + column_types (Sequence[int]): An optional sequence of types (values + of the `ColumnType` enumeration) which the columns of the + result set will be coerced to. Returns: Cursor: As a nonstandard DB-API 2.0 extension, this method returns @@ -971,6 +1120,10 @@ def execute(self, sql, parameters=None): ... {'x': 2, 'y': 4}) >>> cursor.fetchall() [[1, 2], [2, 4]] + + >>> cursor.execute("select 1, 2 UNION ALL select ?, ?", [2, 4]]) + >>> cursor.fetchall() + [[1, 2], [2, 4]] """ self._check_closed() self._description = None @@ -982,14 +1135,16 @@ def execute(self, sql, parameters=None): if errmsg: raise InterfaceError(errmsg) - self._execute(operation, sql, parameters) + self._execute(operation, sql, parameters, column_types=column_types) if self._rowcount == -1: self._load_description() # Optional DB API Extension: execute's return value is unspecified. We # return an iterable over the rows, but this isn't portable across DBs. return self - def executemany(self, sql, seq_of_parameters): + def executemany( + self, sql: str, seq_of_parameters: Sequence[Mapping[str, ParameterValue]] | Sequence[Sequence[ParameterValue]] + ) -> None: """Execute the same SQL statement repeatedly with different parameters. This is currently equivalent to calling execute multiple times, once @@ -998,16 +1153,16 @@ def executemany(self, sql, seq_of_parameters): Args: sql (str): The SQL string to execute, as a Python format string of the format expected by `execute`. - seq_of_parameters (Sequence[Mapping[str, Any]]): A sequence of - mappings from parameter names to the values to be bound for - them. The ``sql`` statement will be run once per element in - this sequence. + seq_of_parameters (Sequence[Mapping[str, Any]] | Sequence[Sequence[Any]]): + The ``sql`` statement will be executed once per element in this + sequence, using each successive element as the parameter values + for the corresponding call to `.execute`. """ self._check_closed() for parameters in seq_of_parameters: self.execute(sql, parameters) - def _execute(self, operation, sql, parameters=None): + def _execute(self, operation, sql, parameters=None, *, column_types=None): self._rowcount = -1 if not self._conn._autocommit: @@ -1025,23 +1180,26 @@ def _execute(self, operation, sql, parameters=None): try: # If variable interpolation fails, then translate the exception to # an InterfaceError to signal that it's a client-side problem. - sql = sql % {name: "@" + name for name in parameters} + # If binding by index then no need to modify sql + by_name = hasattr(parameters, "items") + if by_name: + sql = sql % {name: "@" + name for name in parameters} except KeyError as keyerr: msg = "No value provided for parameter %s" % keyerr - six.raise_from(InterfaceError(msg), keyerr) + raise InterfaceError(msg) from keyerr except Exception as exc: msg = "Invalid Python format string for query" - six.raise_from(InterfaceError(msg), exc) + raise InterfaceError(msg) from exc if _operation_ends_transaction(operation): self._conn._in_transaction = False # txn ends, even on failure try: - self._hndl.execute(sql, parameters) + self._hndl.execute(sql, parameters, column_types=column_types) except cdb2.Error as e: _raise_wrapped_exception(e) - if operation == 'begin': + if operation == "begin": self._conn._in_transaction = True # txn successfully started elif not self._conn._in_transaction and _modifies_rows(operation): # We're not in a transaction, and the last statement could have @@ -1050,11 +1208,11 @@ def _execute(self, operation, sql, parameters=None): # an explicit transaction. We can get the count of affected rows. self._update_rowcount() - def setinputsizes(self, sizes): + def setinputsizes(self, sizes: Sequence[Any]) -> None: """No-op; implemented for PEP-249 compliance.""" self._check_closed() - def setoutputsize(self, size, column=None): + def setoutputsize(self, size: Any, column: int = None) -> None: """No-op; implemented for PEP-249 compliance.""" self._check_closed() @@ -1067,12 +1225,14 @@ def _update_rowcount(self): def _load_description(self): names = self._hndl.column_names() types = self._hndl.column_types() - self._description = tuple((name, type, None, None, None, None, None) - for name, type in zip(names, types)) + self._description = tuple( + (name, type, None, None, None, None, None) + for name, type in zip(names, types) + ) if not self._description: self._description = None - def fetchone(self): + def fetchone(self) -> Row | None: """Fetch the next row of the current result set. Returns: @@ -1087,7 +1247,7 @@ def fetchone(self): except StopIteration: return None - def fetchmany(self, n=None): + def fetchmany(self, n: int | None = None) -> List[Row]: """Fetch the next set of rows of the current result set. Args: @@ -1107,7 +1267,7 @@ def fetchmany(self, n=None): n = self._arraysize return [x for x in itertools.islice(self, 0, n)] - def fetchall(self): + def fetchall(self) -> List[Row]: """Fetch all remaining rows of the current result set. Returns: @@ -1120,7 +1280,7 @@ def fetchall(self): return [x for x in self] # Optional DB API Extension - def __iter__(self): + def __iter__(self) -> Iterator[Row]: """Iterate over all rows in a result set. By default each row is returned as a `list`, where the elements in the diff --git a/comdb2/dbapi2.pyi b/comdb2/dbapi2.pyi deleted file mode 100644 index f8db95f..0000000 --- a/comdb2/dbapi2.pyi +++ /dev/null @@ -1,198 +0,0 @@ -# Copyright 2021 Bloomberg Finance L.P. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import datetime - -from typing import Any -from typing import Callable -from typing import Iterator -from typing import List -from typing import Mapping -from typing import Optional -from typing import Sequence -from typing import Text -from typing import Tuple -from typing import Union - -from . import cdb2 - -apilevel: str -threadsafety: int -paramstyle: str - -STRING: object -BINARY: object -NUMBER: object -DATETIME: object -ROWID: object - - -def Binary(string: Union[Text, bytes]) -> bytes: ... - - -Datetime = datetime.datetime -DatetimeUs = cdb2.DatetimeUs -Timestamp = Datetime -TimestampUs = DatetimeUs - -DatetimeFromTicks = Datetime.fromtimestamp -DatetimeUsFromTicks = DatetimeUs.fromtimestamp -TimestampFromTicks = Timestamp.fromtimestamp -TimestampUsFromTicks = TimestampUs.fromtimestamp - -Value = cdb2.Value -Row = cdb2.Row - - -class Error(Exception): - ... - - -class Warning(Exception): - ... - - -class InterfaceError(Error): - ... - - -class DatabaseError(Error): - ... - - -class InternalError(DatabaseError): - ... - - -class OperationalError(DatabaseError): - ... - - -class ProgrammingError(DatabaseError): - ... - - -class IntegrityError(DatabaseError): - ... - - -class UniqueKeyConstraintError(IntegrityError): - ... - - -class ForeignKeyConstraintError(IntegrityError): - ... - - -class NonNullConstraintError(IntegrityError): - ... - - -class DataError(DatabaseError): - ... - - -class NotSupportedError(DatabaseError): - ... - - -def connect( - database_name: Union[Text, bytes], - tier: Union[Text, bytes] = ..., - autocommit: bool = ..., - host: Optional[Union[Text, bytes]] = ..., -) -> Connection: ... - - -class Connection: - def __init__( - self, - database_name: Union[Text, bytes], - tier: Union[Text, bytes] = ..., - autocommit: bool = ..., - host: Optional[Union[Text, bytes]] = ..., - ) -> None: ... - - @property - def row_factory(self) -> Callable[[List[Text]], Callable[[List[Value]], Row]]: ... - - @row_factory.setter - def row_factory( - self, value: Callable[[List[Text]], Callable[[List[Value]], Row]] - ) -> None: ... - - def close(self, *, ack_current_event=...) -> None: ... - - def commit(self) -> None: ... - - def rollback(self) -> None: ... - - def cursor(self) -> Cursor: ... - - Error = Error - Warning = Warning - InterfaceError = InterfaceError - DatabaseError = DatabaseError - InternalError = InternalError - OperationalError = OperationalError - ProgrammingError = ProgrammingError - IntegrityError = IntegrityError - DataError = DataError - NotSupportedError = NotSupportedError - - -class Cursor: - @property - def arraysize(self) -> int: ... - - @arraysize.setter - def arraysize(self, value: int) -> None: ... - - @property - def description(self) -> Tuple[Text, object, None, None, None, None, None]: ... - - @property - def rowcount(self) -> int: ... - - @property - def connection(self) -> Connection: ... - - def close(self) -> None: ... - - def callproc( - self, - procname: Union[str, Text], - parameters: Sequence[Value], - ) -> Sequence[Value]: ... - - def execute( - self, - sql: Union[str, Text], - parameters: Optional[Mapping[Text, Value]] = None, - ) -> Cursor: ... - - def executemany( - self, - sql: str, - seq_of_parameters: Sequence[Mapping[Text, Value]], - ) -> None: ... - - def setinputsizes(self, sizes: Sequence[Any]) -> None: ... - - def setoutputsize(self, size: Any, column: int = ...) -> None: ... - - def fetchone(self) -> Optional[Row]: ... - - def fetchmany(self, n: Optional[int] = ...) -> Sequence[Row]: ... - - def fetchall(self) -> Sequence[Row]: ... - - def __iter__(self) -> Iterator[Row]: ... diff --git a/comdb2/factories.py b/comdb2/factories.py index c63e5f8..363111d 100644 --- a/comdb2/factories.py +++ b/comdb2/factories.py @@ -21,12 +21,17 @@ A factory function will be called with a list of column names, and must return a callable that will be called once per row with a list of column values. """ -from __future__ import unicode_literals + +from __future__ import annotations + from collections import namedtuple from collections import Counter +from .cdb2 import Value +from collections.abc import Callable +from typing import NamedTuple -def namedtuple_row_factory(col_names): +def namedtuple_row_factory(col_names: list[str]) -> Callable[[list[Value]], NamedTuple]: """Return each result row as a `collections.namedtuple`. The fields of the `~collections.namedtuple` are set to the names of the @@ -75,10 +80,10 @@ def namedtuple_row_factory(col_names): # Ensure DML doesn't raise an exception for an invalid column name if len(col_names) == 1: if col_names[0] in ("rows inserted", "rows updated", "rows deleted"): - return namedtuple('Row', col_names, rename=True)._make + return namedtuple("Row", col_names, rename=True)._make try: - return namedtuple('Row', col_names)._make + return namedtuple("Row", col_names)._make except ValueError: # If the error was caused by duplicated column names, raise a more # preceise error message. Otherwise, re-raise. @@ -86,7 +91,7 @@ def namedtuple_row_factory(col_names): raise -def dict_row_factory(col_names): +def dict_row_factory(col_names: list[str]) -> Callable[[list[Value]], dict[str, Value]]: """Return each result row as a `dict` mapping column names to values. Note: @@ -118,8 +123,10 @@ def dict_row_factory(col_names): 1 """ _raise_on_duplicate_column_names(col_names) + def dict_row(col_vals): return dict(zip(col_names, col_vals)) + return dict_row diff --git a/comdb2/factories.pyi b/comdb2/factories.pyi deleted file mode 100644 index 7ca9edf..0000000 --- a/comdb2/factories.pyi +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2021 Bloomberg Finance L.P. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Callable -from typing import Dict -from typing import NamedTuple -from typing import List -from typing import Text - -from .cdb2 import Value - -def namedtuple_row_factory( - col_names: List[Text], -) -> Callable[[List[Value]], NamedTuple]: ... - -def dict_row_factory( - col_names: List[Text] -) -> Callable[[List[Value]], Dict[Text, Value]]: ... diff --git a/docs/cdb2.rst b/docs/cdb2.rst index 83ff082..134a52a 100644 --- a/docs/cdb2.rst +++ b/docs/cdb2.rst @@ -46,8 +46,12 @@ Enumerations Several mappings are provided to expose enumerations from ``cdb2api.h`` to Python. - .. autodata:: TYPE +.. autoclass:: ColumnType - .. autodata:: HANDLE_FLAGS +.. autodata:: TYPE - .. autodata:: ERROR_CODE +.. autoclass:: ConnectionFlags + +.. autodata:: HANDLE_FLAGS + +.. autodata:: ERROR_CODE diff --git a/docs/conf.py b/docs/conf.py index c6c59ec..ed54495 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Comdb2 documentation build configuration file, created by # sphinx-quickstart on Thu Sep 8 16:16:10 2016. @@ -14,59 +13,60 @@ import sys import os -sys.path.insert(0, os.path.abspath('..')) + +sys.path.insert(0, os.path.abspath("..")) from comdb2 import __version__ # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.intersphinx', - 'sphinx.ext.napoleon', - 'sphinx.ext.viewcode', - 'sphinx.ext.githubpages', + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", + "sphinx.ext.napoleon", + "sphinx.ext.viewcode", + "sphinx.ext.githubpages", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['templates'] +templates_path = ["templates"] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The root toctree document. -root_doc = 'index' +root_doc = "index" # General information about the project. -project = u'Comdb2' -copyright = u'2017, Bloomberg LP' +project = "Comdb2" +copyright = "2017, Bloomberg LP" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # -version = '.'.join(__version__.split('.')[:2]) +version = ".".join(__version__.split(".")[:2]) # The full version, including alpha/beta/rc tags. release = __version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -74,58 +74,58 @@ # The reST default role (used for this markup: `text`) to use for all # documents. -default_role = 'py:obj' +default_role = "py:obj" # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. -modindex_common_prefix = ['comdb2.'] +modindex_common_prefix = ["comdb2."] # If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False +# keep_warnings = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, @@ -135,51 +135,51 @@ # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +# html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'Comdb2doc' +htmlhelp_basename = "Comdb2doc" # -- Options for LaTeX output --------------------------------------------- @@ -205,23 +205,23 @@ # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output --------------------------------------- @@ -234,7 +234,7 @@ # ] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------- @@ -249,27 +249,23 @@ # ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +# texinfo_no_detailmenu = False # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'python': ('https://docs.python.org/3', None), - 'six': ('https://six.readthedocs.io', None)} - -# Don't highlight the code for the Python standard library datetime module. -viewcode_import = False +intersphinx_mapping = {"python": ("https://docs.python.org/3", None)} # Don't automatically infer types from type annotations. -autodoc_typehints = 'none' +autodoc_typehints = "none" # Replace the extension module with a mock. -autodoc_mock_imports = ['comdb2._ccdb2'] +autodoc_mock_imports = ["comdb2._ccdb2"] diff --git a/docs/dbapi2.rst b/docs/dbapi2.rst index 9ad9692..f5d87e8 100644 --- a/docs/dbapi2.rst +++ b/docs/dbapi2.rst @@ -87,9 +87,9 @@ them if you so choose, but you are not required to. Creates an object suitable for binding as a BLOB parameter. - If the input argument was a `six.text_type` Unicode string, it is - encoded as a UTF-8 byte string and returned. Otherwise, the input - argument is passed to the `bytes` constructor, and the result returned. + If the input argument was a `str` object, it is encoded as a UTF-8 byte + string and returned. Otherwise, the input argument is passed to the + `bytes` constructor, and the result returned. :param string: A string from which the new object is constructed :rtype: `bytes` @@ -124,6 +124,11 @@ for consistency with the required DATETIME constructors documented above. after the Unix epoch, with microsecond precision :rtype: `DatetimeUs` +.. class:: ColumnType + + This is an alias for `comdb2.cdb2.ColumnType`, reexported from + ``comdb2.dbapi2`` for convenience. + .. _Exceptions: Exceptions diff --git a/docs/tips.rst b/docs/tips.rst index e5c9fdf..d2ba8c4 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -2,17 +2,16 @@ Best Practices, Tips, and Tricks ******************************** -#. This package uses `six.text_type` for text and `bytes` for blobs; this may - be surprising if you're using `str` in Python 2. See :doc:`types` for an - explanation of what Python types must be used when binding parameters, and - what Python types will be returned for result columns. +#. This package uses `str` for text and `bytes` for blobs. See :doc:`types` for + an explanation of what Python types must be used when binding parameters, + and what Python types will be returned for result columns. An error message saying "incompatible values from SQL blob of length 3 to bcstring field 'foo'" most likely means that you passed a byte string where a unicode string should have been passed. An error message saying "incompatible values from SQL string of length 3 to - bbytearray field 'bar'" most likely means that you passed a unicode string + bytearray field 'bar'" most likely means that you passed a unicode string where a byte string should have been passed. Note the byte string must be *exactly* the length of the column. No padding is performed automatically, unless the column is declared with a ``dbpad`` attribute in the csc2 schema @@ -24,11 +23,8 @@ Best Practices, Tips, and Tricks a ``WHERE`` clause that you expected to match some rows will silently fail to match any. -#. Prefer using Python 3 rather than Python 2 when possible. The above type - mappings are much more intuitive in Python 3 than in Python 2. If you can't - use Python 3 but are writing a new module, prefer to use ``from __future__ - import unicode_literals`` to opt into forward compatible unicode string - literals (rather than the default, string literals as byte strings). +#. The latest version of this package only supports Python 3. If you can't + use Python 3, make sure to use version less than ``1.5.0``. #. The database can time out connections that have been idle for a period of time, and each idle connection uses some amount of resources on the database @@ -60,19 +56,41 @@ Best Practices, Tips, and Tricks Because of this: https://xkcd.com/327/ + When using comdb2 R8, you can even bind a `list` or `tuple` object like so:: + + c.execute( + "SELECT %(needle)s IN CARRAY(%(haystack)s)", + {'needle': 5, 'haystack': [1, 2, 3, 4, 5]} + ) + .. note:: The two modules provided by this package use different syntax for SQL placeholders. See `.dbapi2.Cursor.execute` and `.cdb2.Handle.execute` for details. -#. For `.dbapi2`, be sure to escape any ``%`` signs in a query by doubling - them. That is, instead of:: +#. When using `.dbapi2` but not parameter binding by position be sure to escape + any literal ``%`` signs in a query by doubling them. That is, instead of:: - c.execute("select * from tbl where col like 'foo%'") + cursor.execute( + "select * from log where msg like 'ERROR: %' and pid = %(pid)s", + {"pid": 1234}, + ) You need to write:: - c.execute("select * from tbl where col like 'foo%%'") + cursor.execute( + "select * from log where msg like 'ERROR: %%' and pid = %(pid)s", + {"pid": 1234}, + ) + + Also, if you don't want to pass any parameters to a `.dbapi2` query, it's + better to pass ``()`` than the default ``None`` value, because this removes + the need to escape any literal ``%`` signs that appear in the query:: + + cursor.execute( + "select * from log where msg like 'ERROR: %'", + (), + ) See `.dbapi2.paramstyle` for an explanation of why. @@ -83,8 +101,23 @@ Best Practices, Tips, and Tricks `.cdb2.Handle` (any isolation level higher than ``READ COMMITTED`` would obviously work as well). -#. The underlying API doesn't currently allow binding lists. The following snippet - will be useful for a ``$var in $list`` query with a dynamically generated list:: +#. When using comdb2 R8, the library supports binding of (non-empty) `list` or + `tuple` objects with the help of the ``CARRAY`` function, as long as all + elements of the sequence are of the same type:: + + from comdb2.dbapi2 import connect + + def search_in_list(needle, haystack): + params = {'needle': needle, 'haystack': haystack} + sql = "select %(needle)s in CARRAY(%(haystack)s)" + print(connect('mattdb').cursor().execute(sql, params).fetchall()) + + haystack = [1, 2, 3, 4, 5] + search_in_list(0, haystack) + search_in_list(5, haystack) + + If you still need to use R7, you would want to generate the query + dynamically, for example:: from comdb2.dbapi2 import connect diff --git a/docs/types.rst b/docs/types.rst index 0fd1da9..d452022 100644 --- a/docs/types.rst +++ b/docs/types.rst @@ -14,10 +14,11 @@ SQL type Python type NULL ``None`` integer `int` real `float` -blob `six.binary_type` (aka `bytes` in Python 3, ``str`` in Python 2) -text `six.text_type` (aka `str` in Python 3, ``unicode`` in Python 2) +blob `bytes` +text `str` datetime `datetime.datetime` datetimeus `~.cdb2.DatetimeUs` +carray `list` or `tuple` of exactly one of the types above (R8 only) intervalym not supported intervalds not supported intervaldsus not supported @@ -42,47 +43,11 @@ into a column of type ``short``), an exception will be raised. String and Blob Types ===================== -This module uses byte strings to represent BLOB values, and Unicode strings to -represent TEXT columns. This was chosen for maximum forward compatibility with -Python 3, and to make it easier to write code that will work identically in -both languages. This decision has many important ramifications. +You may occasionally need to convert between byte strings and Unicode strings: -#. In Python 2, ``'foo'`` is the wrong type for binding a ``cstring`` column. - You instead need to use a Unicode literal like ``u'foo'``. - Alternately, you can make all string literals in your module be Unicode - literals by default by doing:: - - from __future__ import unicode_literals - - .. note:: - It's not safe to blindly do this in existing code as it changes that - code's behavior, and might cause problems. - -#. If you have a variable of type `six.binary_type` (byte string) and you - want to pass it to the database as a TEXT value, you need to convert it to - a `six.text_type` (Unicode) string using `~bytes.decode`. In Python 2, - unless ``unicode_literals`` was imported from `__future__`, you will need to - do this with every string variable that you want to use as a cstring - column. For example:: - - value = 'foo' - params = {'col': value.decode('utf-8')} - # dbapi2 - cursor.execute("select * from tbl where col=%(col)s", params) - # cdb2 - handle.execute("select * from tbl where col=@col", params) - -#. TEXT columns are returned to you as `six.text_type` (Unicode) strings. - If you need to pass them to a library that expects `six.binary_type` (byte) - strings, you have to use `~str.encode`, like this:: - - cursor.execute("select col from tbl") - for row in cursor: - col = row[0] - library_func(col.encode('utf-8')) - - This may come up a lot in Python 2, where historically libraries were - written to expect byte strings rather than Unicode strings. +#. If you have a variable of type `bytes` and you want to pass it to the + database as a TEXT value, you need to convert it to a `str` using + `~bytes.decode`. #. When a TEXT column is read back from the database, it is decoded as UTF-8. If your database has non-UTF-8 text in a ``cstring`` column, you need to @@ -91,14 +56,14 @@ both languages. This decision has many important ramifications. cursor.execute("select cast(col as blob) from tbl") - That will result in you receiving a byte string rather than a Unicode - string for that result column. + That will result in you receiving a `bytes` object rather than `str` for + that result column. .. note:: ASCII is a subset of UTF-8, so this is not required for if the column contains plain ASCII text. -#. Likewise, when a Unicode string is sent to the database, it is encoded as +#. Likewise, when a `str` object is sent to the database, it is encoded as UTF-8. If you need to store non-UTF-8 text in a ``cstring`` column in your database, you need to bind a byte string in Python land, and then cast the value from BLOB to TEXT in SQL. For instance, to bind a latin1 string @@ -126,9 +91,8 @@ Whenever either type of datetime is retrieved as a result column, the returned `datetime.datetime` object will be aware, meaning it will be associated with a timezone (namely, the current timezone of the Comdb2 connection). Whenever either type of datetime is sent as a query parameter, it is permitted to be -naïve, in which case it uses the current timezone of the Comdb2 connection. By -default that timezone is UTC but it can be changed on a per-connection basis if -needed. +naïve, in which case it is treated as UTC (and *not* as the system's local +time, which is the more common interpretation in newer Python libraries). Comdb2 additionally supports three types for representing time intervals. One represents an exact number of elapsed milliseconds between two times @@ -147,3 +111,25 @@ string columns. That currently makes it impossible to map them to Python's `decimal.Decimal` type as we would like to. If ``libcdb2api`` is ever changed to properly distinguish between DECIMAL and TEXT columns this package will be enhanced to properly expose DECIMAL columns. + +C Array Types +============= + +.. note:: + This only works with comdb2 R8. If you need to connect to a database that is + still running comdb2 R7, the queries using this feature will fail. + +Often in SQL queries, it is useful to bind an array of elements of the same +type, for example when using the ``IN`` operator. In this case, you can use a +(non-empty) `list` or `tuple`, as long as all elements of the sequence are of +the same type (and one of the above Python types). Note that nested sequences +are not allowed. + +Note that when binding an element of that type, you want to use the ``CARRAY`` +function, like so:: + + params = {'arr': [1, 2, 3, 4]} + # dbapi2 + c.execute("select 2 in CARRAY(%(arr)s)", params) + # cdb2 + h.execute("select 2 in CARRAY(@arr)", params) diff --git a/setup.py b/setup.py index ad78159..f0fd474 100644 --- a/setup.py +++ b/setup.py @@ -9,47 +9,47 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys import pkgconfig from setuptools import setup, Extension about = {} -with open('comdb2/_about.py') as fp: +with open("comdb2/_about.py") as fp: exec(fp.read(), about) def make_static_extension(name, **kwargs): - libraries = kwargs.pop('libraries', []) - pkg_config = pkgconfig.parse(' '.join(libraries), static=True) - libraries = pkg_config['libraries'] - library_dirs = pkg_config['library_dirs'] - include_dirs = kwargs.pop("include_dirs", []) + pkg_config['include_dirs'] + libraries = kwargs.pop("libraries", []) + pkg_config = pkgconfig.parse(" ".join(libraries), static=True) + libraries = pkg_config["libraries"] + library_dirs = pkg_config["library_dirs"] + include_dirs = kwargs.pop("include_dirs", []) + pkg_config["include_dirs"] return Extension( name, libraries=libraries, library_dirs=library_dirs, include_dirs=include_dirs, - **kwargs + **kwargs, ) ccdb2 = make_static_extension( "comdb2._ccdb2", - extra_compile_args=['-std=c99'], - libraries=['cdb2api'], - sources=["comdb2/_ccdb2.pyx"] + language="c++", + libraries=["cdb2api"], + sources=["comdb2/_ccdb2.pyx"], ) setup( - name='comdb2', + name="comdb2", version=about["__version__"], - author='Alex Chamberlain', - author_email='achamberlai9@bloomberg.net', - packages=['comdb2'], - install_requires=["six", "pytz"], - extras_require={"tests": ["python-dateutil>=2.6.0", "pytest", "mock; python_version < '3.3'"]}, + author="Alex Chamberlain", + author_email="achamberlai9@bloomberg.net", + packages=["comdb2"], + install_requires=["pytz; python_version < '3.14'"], + extras_require={"tests": ["python-dateutil>=2.6.0", "pytest"]}, + python_requires=">=3.8", ext_modules=[ccdb2], package_data={"comdb2": ["py.typed", "*.pyi"]}, ) diff --git a/tests/test_cdb2.py b/tests/test_cdb2.py index 6620baa..3835705 100644 --- a/tests/test_cdb2.py +++ b/tests/test_cdb2.py @@ -9,23 +9,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import unicode_literals, absolute_import +import datetime from comdb2 import cdb2 from comdb2.factories import dict_row_factory from comdb2.factories import namedtuple_row_factory import pytest -import six -COLUMN_LIST = ("short_col u_short_col int_col u_int_col longlong_col" - " float_col double_col byte_col byte_array_col" - " cstring_col pstring_col blob_col datetime_col vutf8_col" - ).split() +COLUMN_LIST = ( + "short_col u_short_col int_col u_int_col longlong_col" + " float_col double_col byte_col byte_array_col" + " cstring_col pstring_col blob_col datetime_col vutf8_col" +).split() @pytest.fixture(autouse=True) def delete_all_rows(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") while True: hndl.execute("delete from all_datatypes limit 100") if hndl.get_effects()[0] != 100: @@ -38,17 +38,19 @@ def delete_all_rows(): def test_garbage_collecting_unused_handle(): - cdb2.Handle('mattdb', 'dev').execute("select 1 union select 2") + cdb2.Handle("mattdb", "dev").execute( + "select /*CURSOR DRAIN TEST*/ 1 union select 2" + ) def test_commit_on_unused_connection(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") with pytest.raises(cdb2.Error): hndl.execute("commit") def test_empty_transactions(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") hndl.execute("begin") hndl.execute("commit") @@ -57,17 +59,35 @@ def test_empty_transactions(): def test_binding_parameters(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") hndl.execute("insert into simple(key, val) values(@k, @v)", dict(v=2, k=1)) hndl.execute("insert into simple(key, val) values(@k, @v)", dict(k=3, v=4)) assert hndl.get_effects()[0] == 1 + hndl.execute("insert into simple(key, val) values(?, ?)", [5, 6]) + assert hndl.get_effects()[0] == 1 + rows = list(hndl.execute("select key, val from simple order by key")) - assert rows == [[1, 2], [3, 4]] + assert rows == [[1, 2], [3, 4], [5, 6]] + + +def test_binding_no_parameters(): + hndl = cdb2.Handle("mattdb", "dev") + rows = list(hndl.execute("select 1 % 10")) + assert rows == [[1]] + + rows = list(hndl.execute("select 2 % 10", {})) + assert rows == [[2]] + + rows = list(hndl.execute("select 3 % 10", ())) + assert rows == [[3]] + + rows = list(hndl.execute("select 4 % 10", [])) + assert rows == [[4]] def test_commit_failures(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") hndl.execute("begin") hndl.execute("insert into simple(key, val) values(@k, @v)", dict(k=1, v=2)) hndl.execute("insert into simple(key, val) values(@k, @v)", dict(k=3)) @@ -75,106 +95,106 @@ def test_commit_failures(): try: hndl.execute("commit") except cdb2.Error as exc: - assert exc.error_code == cdb2.ERROR_CODE['PREPARE_ERROR'] + assert exc.error_code == cdb2.ERROR_CODE["PREPARE_ERROR"] else: assert False # an assertion should have been raised! def test_error_from_closing_connection_twice(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") hndl.close() with pytest.raises(cdb2.Error): hndl.close() def test_iterating_newly_initialized_handle(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") for row in hndl: assert False def test_timezone_handling(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") rows = list(hndl.execute("select now()")) assert len(rows) == 1 - assert rows[0][0].tzname() == 'UTC' + assert rows[0][0].tzname() == "UTC" - hndl = cdb2.Handle('mattdb', 'dev', tz='GMT') + hndl = cdb2.Handle("mattdb", "dev", tz="GMT") rows = list(hndl.execute("select now()")) assert len(rows) == 1 - assert rows[0][0].tzname() == 'GMT' + assert rows[0][0].tzname() == "GMT" hndl.execute("set timezone UTC") rows = list(hndl.execute("select now()")) assert len(rows) == 1 - assert rows[0][0].tzname() == 'UTC' + assert rows[0][0].tzname() == "UTC" def test_passing_flags(): # Bad SQL statements in a transaction fail early - flags = cdb2.HANDLE_FLAGS['READ_INTRANS_RESULTS'] + flags = cdb2.HANDLE_FLAGS["READ_INTRANS_RESULTS"] # Test passing flags - hndl = cdb2.Handle('mattdb', 'dev', flags=flags) - hndl.execute('begin') + hndl = cdb2.Handle("mattdb", "dev", flags=flags) + hndl.execute("begin") with pytest.raises(cdb2.Error): - hndl.execute('foobar') + hndl.execute("foobar") # Default behavior, without any flags passed - hndl = cdb2.Handle('mattdb', 'dev', flags=0) - hndl.execute('begin') - hndl.execute('foobar') + hndl = cdb2.Handle("mattdb", "dev", flags=0) + hndl.execute("begin") + hndl.execute("foobar") with pytest.raises(cdb2.Error): hndl.execute("select 1") def test_row_factories(): query = "select 1 as 'a', 2 as 'b' union select 3, 4 order by a" - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") assert list(hndl.execute(query)) == [[1, 2], [3, 4]] hndl.row_factory = dict_row_factory assert hndl.row_factory == dict_row_factory - assert list(hndl.execute(query)) == [{'a': 1, 'b': 2}, {'a': 3, 'b': 4}] + assert list(hndl.execute(query)) == [{"a": 1, "b": 2}, {"a": 3, "b": 4}] hndl.row_factory = namedtuple_row_factory assert hndl.row_factory == namedtuple_row_factory rows = [r._asdict() for r in hndl.execute(query)] - assert rows == [{'a': 1, 'b': 2}, {'a': 3, 'b': 4}] + assert rows == [{"a": 1, "b": 2}, {"a": 3, "b": 4}] def test_row_factories_with_dup_col_names(): query = "select 1 as 'a', 2 as 'a'" - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") assert list(hndl.execute(query)) == [[1, 2]] hndl.row_factory = namedtuple_row_factory with pytest.raises(cdb2.Error) as exc_info: hndl.execute(query) - assert isinstance(exc_info.value.args[1], six.text_type) + assert isinstance(exc_info.value.args[1], str) hndl.row_factory = dict_row_factory with pytest.raises(cdb2.Error) as exc_info: hndl.execute(query) - assert isinstance(exc_info.value.args[1], six.text_type) + assert isinstance(exc_info.value.args[1], str) def test_setting_non_callable_row_factory(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") with pytest.raises(TypeError): hndl.row_factory = "foo" def test_unsetting_row_factory(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") assert list(hndl.execute("select 1")) == [[1]] hndl.row_factory = dict_row_factory - assert list(hndl.execute("select 1")) == [{'1': 1}] + assert list(hndl.execute("select 1")) == [{"1": 1}] hndl.row_factory = None assert list(hndl.execute("select 1")) == [[1]] @@ -184,20 +204,21 @@ def test_failures_instantiating_row_class(): def factory(col_names): def klass(col_values): raise TypeError(col_values) + return klass query = "select 1 as 'a', 2 as 'a'" - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") hndl.row_factory = factory it = iter(hndl.execute(query)) with pytest.raises(cdb2.Error) as exc_info: next(it) - assert isinstance(exc_info.value.args[1], six.text_type) + assert isinstance(exc_info.value.args[1], str) def test_get_effects(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") hndl.execute("select 1 union all select 2") assert hndl.get_effects() == (0, 2, 0, 0, 0) @@ -222,45 +243,46 @@ def test_get_effects(): @pytest.mark.xfail(reason="cdb2_open ignores tier") def test_nonascii_error_messages(): with pytest.raises(cdb2.Error) as exc_info: - cdb2.Handle('mattdb', b'\xc3') - assert r'cluster type \xc3.' in exc_info.value.error_message + cdb2.Handle("mattdb", b"\xc3") + assert r"cluster type \xc3." in exc_info.value.error_message - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") with pytest.raises(cdb2.Error) as exc_info: - hndl.execute(b'select \xC3') - assert r'no such column: \xc3' in exc_info.value.error_message + hndl.execute(b"select \xc3") + assert r"no such column: \xc3" in exc_info.value.error_message teststr = bytes(bytearray(range(128, 256))) with pytest.raises(cdb2.Error) as exc_info: - hndl.execute(b'select ' + teststr) + hndl.execute(b"select " + teststr) assert ( - r'no such column: ' - r'\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f' - r'\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f' - r'\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf' - r'\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf' - r'\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf' - r'\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf' - r'\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef' - r'\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff' - in exc_info.value.error_message) + r"no such column: " + r"\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f" + r"\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f" + r"\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf" + r"\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf" + r"\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf" + r"\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf" + r"\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef" + r"\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" + in exc_info.value.error_message + ) teststr = bytes(bytearray(range(1, 31))) with pytest.raises(cdb2.Error) as exc_info: - cdb2.Handle('mattdb', teststr) - errmsg_utf8 = exc_info.value.error_message.encode('utf-8') + cdb2.Handle("mattdb", teststr) + errmsg_utf8 = exc_info.value.error_message.encode("utf-8") assert teststr in errmsg_utf8 - teststr = bytes(bytearray(range(1, 31))) + b'\xff' + teststr = bytes(bytearray(range(1, 31))) + b"\xff" with pytest.raises(cdb2.Error) as exc_info: - cdb2.Handle('mattdb', teststr) - errmsg_utf8 = exc_info.value.error_message.encode('utf-8') + cdb2.Handle("mattdb", teststr) + errmsg_utf8 = exc_info.value.error_message.encode("utf-8") assert teststr not in errmsg_utf8 - assert teststr.decode('latin1').encode('unicode_escape') in errmsg_utf8 + assert teststr.decode("latin1").encode("unicode_escape") in errmsg_utf8 def test_namedtuple_factory_dml(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") hndl.row_factory = namedtuple_row_factory hndl.execute("insert into simple (key, val) values (1, 1)") @@ -274,7 +296,7 @@ def test_namedtuple_factory_dml(): def test_parameter_name_in_binding_errors_exception(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") class BadBytes(bytes): def __len__(self): @@ -292,11 +314,51 @@ def __repr__(self): def test_parameter_name_in_binding_errors_noexception(): - hndl = cdb2.Handle('mattdb', 'dev') + hndl = cdb2.Handle("mattdb", "dev") with pytest.raises(Exception) as exc: - hndl.execute("select @param", dict(param=(5+6j))) + hndl.execute("select @param", dict(param=(5 + 6j))) assert exc.value.args[1] == ( "Can't map complex value (5+6j) for parameter 'param' to a Comdb2 type" ) + + +def test_specifying_column_types(): + # GIVEN + hndl = cdb2.Handle("mattdb", "dev") + column_types = [ + cdb2.ColumnType.INT, + cdb2.ColumnType.REAL, + cdb2.ColumnType.TEXT, + cdb2.ColumnType.BLOB, + cdb2.ColumnType.DATETIME, + cdb2.ColumnType.DATETIMEUS, + ] + sql = "select 0, 0, 0, 0, 0, 0" + + # WHEN + (row,) = hndl.execute(sql, column_types=column_types) + + # THEN + assert row is not None + assert type(row[0]) is int + assert type(row[1]) is float + assert type(row[2]) is str + assert type(row[3]) is bytes + assert type(row[4]) is datetime.datetime + assert type(row[5]) is cdb2.DatetimeUs + + +def test_providing_empty_column_types_array(): + # GIVEN + hndl = cdb2.Handle("mattdb", "dev") + column_types = [] + sql = "select 0" + + # WHEN + (row,) = hndl.execute(sql, column_types=column_types) + + # THEN + assert row is not None + assert type(row[0]) is int diff --git a/tests/test_cdb2_datetimeus.py b/tests/test_cdb2_datetimeus.py index d1a9462..c611fb6 100644 --- a/tests/test_cdb2_datetimeus.py +++ b/tests/test_cdb2_datetimeus.py @@ -9,10 +9,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import unicode_literals, absolute_import from comdb2 import cdb2 -import pytz import dateutil.tz import pytest @@ -85,21 +83,34 @@ def test_datetimeus_fromtimestamp(): def test_datetimeus_astimezone(): - eastern = pytz.timezone('US/Eastern') - loc_dt = eastern.localize(datetime.datetime(2002, 10, 27, 6, 0, 0)) + eastern = dateutil.tz.gettz("US/Eastern") + loc_dt = datetime.datetime(2002, 10, 27, 6, 0, 0, tzinfo=eastern) dtu = cdb2.DatetimeUs.fromdatetime(loc_dt) - assert type(dtu.astimezone(pytz.utc)) == cdb2.DatetimeUs + assert type(dtu.astimezone(datetime.timezone.utc)) == cdb2.DatetimeUs + + +def test_datetimeus_as_datetime_naive(): + dtu = cdb2.DatetimeUs.fromtimestamp(time.time()) + dt = dtu.as_datetime() + assert type(dt) == datetime.datetime + assert dt == dtu + + +def test_datetimeus_as_datetime_with_tz(): + dtu = cdb2.DatetimeUs.fromtimestamp(time.time(), tz=datetime.timezone.utc) + dt = dtu.as_datetime() + assert type(dt) == datetime.datetime + assert dt == dtu def test_datetimeus_type_stickiness(): def check(obj): assert isinstance(obj, cdb2.DatetimeUs) - new_york = pytz.timezone('America/New_York') - utc = pytz.UTC + new_york = dateutil.tz.gettz("America/New_York") + utc = datetime.timezone.utc - check(new_york.localize(cdb2.DatetimeUs(2016, 8, 15, 18, 47, 15, 123456))) check(cdb2.DatetimeUs(2016, 8, 15, 18, 47, 15, 123456, new_york)) check(cdb2.DatetimeUs(2016, 8, 15, 18, 47, 15, 123456)) check(cdb2.DatetimeUs.today()) @@ -115,21 +126,22 @@ def check(obj): check(cdb2.DatetimeUs.now() - datetime.timedelta(0)) check(cdb2.DatetimeUs.now() + datetime.timedelta(0)) check(datetime.timedelta(0) + cdb2.DatetimeUs.now()) - check(cdb2.DatetimeUs.now().replace(year=2015)) + check(cdb2.DatetimeUs.now().replace(year=2016)) check(cdb2.DatetimeUs.now().replace(tzinfo=new_york)) check(cdb2.DatetimeUs.now(utc).astimezone(new_york)) -@pytest.mark.skipif(not hasattr(datetime.datetime, 'fold'), - reason='Skipped before PEP 495') +@pytest.mark.skipif( + not hasattr(datetime.datetime, "fold"), reason="Skipped before PEP 495" +) def test_datetimeus_fold(): - NYC = dateutil.tz.gettz('America/New_York') + NYC = dateutil.tz.gettz("America/New_York") dt = datetime.datetime(2004, 10, 31, 1, 30, fold=1, tzinfo=NYC) dtus = cdb2.DatetimeUs.fromdatetime(dt) - assert dtus.fold == 1 # Check that it handles fold correctly + assert dtus.fold == 1 # Check that it handles fold correctly # Make sure that the time is correctly disambiguating - assert dt.tzname() == "EST" # Just in case it's not a DatetimeUs problem + assert dt.tzname() == "EST" # Just in case it's not a DatetimeUs problem assert dtus.tzname() == "EST" diff --git a/tests/test_dbapi2.py b/tests/test_dbapi2.py index 26344f5..a1cbd6e 100644 --- a/tests/test_dbapi2.py +++ b/tests/test_dbapi2.py @@ -9,55 +9,67 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import unicode_literals, absolute_import - -from comdb2.dbapi2 import _sql_operation -from comdb2.dbapi2 import NUMBER -from comdb2.dbapi2 import BINARY -from comdb2.dbapi2 import STRING -from comdb2.dbapi2 import DATETIME -from comdb2.dbapi2 import connect -from comdb2.dbapi2 import Binary -from comdb2.dbapi2 import Datetime -from comdb2.dbapi2 import DatetimeUs -from comdb2.dbapi2 import Timestamp -from comdb2.dbapi2 import TimestampUs -from comdb2.dbapi2 import DataError -from comdb2.dbapi2 import OperationalError -from comdb2.dbapi2 import IntegrityError -from comdb2.dbapi2 import ForeignKeyConstraintError -from comdb2.dbapi2 import NonNullConstraintError -from comdb2.dbapi2 import UniqueKeyConstraintError -from comdb2.dbapi2 import InterfaceError -from comdb2.dbapi2 import NotSupportedError -from comdb2.dbapi2 import ProgrammingError -from comdb2 import cdb2 -from comdb2.factories import dict_row_factory -from comdb2.factories import namedtuple_row_factory -import pytest + import datetime -import pytz -import six +import re from functools import partial +from unittest.mock import patch -try: - from unittest.mock import patch -except ImportError: - from mock import patch - -COLUMN_LIST = ("short_col u_short_col int_col u_int_col longlong_col" - " float_col double_col byte_col byte_array_col" - " cstring_col pstring_col blob_col datetime_col vutf8_col" - ).split() +import pytest +from dateutil.tz import gettz -COLUMN_TYPE = (NUMBER, NUMBER, NUMBER, NUMBER, NUMBER, - NUMBER, NUMBER, BINARY, BINARY, - STRING, STRING, BINARY, DATETIME, STRING) +from comdb2 import cdb2 +from comdb2.dbapi2 import ( + BINARY, + DATETIME, + NUMBER, + STRING, + Binary, + DataError, + Datetime, + DatetimeUs, + ForeignKeyConstraintError, + IntegrityError, + InterfaceError, + NonNullConstraintError, + NotSupportedError, + OperationalError, + ProgrammingError, + Timestamp, + TimestampUs, + UniqueKeyConstraintError, + _sql_operation, + connect, +) +from comdb2.factories import dict_row_factory, namedtuple_row_factory + +COLUMN_LIST = ( + "short_col u_short_col int_col u_int_col longlong_col" + " float_col double_col byte_col byte_array_col" + " cstring_col pstring_col blob_col datetime_col vutf8_col" +).split() + +COLUMN_TYPE = ( + NUMBER, + NUMBER, + NUMBER, + NUMBER, + NUMBER, + NUMBER, + NUMBER, + BINARY, + BINARY, + STRING, + STRING, + BINARY, + DATETIME, + STRING, +) @pytest.fixture(autouse=True) def delete_all_rows(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() while True: cursor.execute("delete from all_datatypes limit 100") @@ -80,37 +92,37 @@ def delete_all_rows(): @pytest.mark.xfail(reason="cdb2_open ignores tier") def test_invalid_cluster(): with pytest.raises(OperationalError): - connect('mattdb', 'foo') + connect("mattdb", "foo") def test_invalid_dbname(): with pytest.raises(OperationalError): - connect('', 'dev') + connect("", "dev") def test_closing_unused_connection(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") conn.close() def test_garbage_collecting_unused_connection(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") del conn def test_commit_on_unused_connection(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") conn.commit() def test_rollback_on_unused_connection(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") conn.rollback() def test_inserts(): # Without params - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("insert into simple(key, val) values(1, 2)") cursor.connection.commit() @@ -120,10 +132,9 @@ def test_inserts(): assert cursor.fetchall() == [[1, 2]] # With params - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() - cursor.execute("insert into simple(key, val) values(%(k)s, %(v)s)", - dict(k=3, v=4)) + cursor.execute("insert into simple(key, val) values(%(k)s, %(v)s)", dict(k=3, v=4)) conn.commit() assert cursor.rowcount == 1 @@ -132,7 +143,7 @@ def test_inserts(): def test_rollback(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("insert into simple(key, val) values(1, 2)") conn.rollback() @@ -142,10 +153,12 @@ def test_rollback(): def test_commit_failures(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() - cursor.executemany("insert into simple(key, val) values(%(key)s, %(val)s)", - [ dict(key=1, val=2), dict(key=3, val=None) ]) + cursor.executemany( + "insert into simple(key, val) values(%(key)s, %(val)s)", + [dict(key=1, val=2), dict(key=3, val=None)], + ) with pytest.raises(IntegrityError): conn.commit() assert cursor.rowcount == 0 @@ -155,7 +168,7 @@ def test_commit_failures(): def test_unique_key_violation(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("insert into simple(key, val) values(1, 2)") conn.commit() @@ -165,42 +178,43 @@ def test_unique_key_violation(): def test_constraint_errors(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("insert into simple(key, val) values(1, 2)") cursor.execute("insert into simple(key, val) values(1, 2)") with pytest.raises(UniqueKeyConstraintError) as exc_info: conn.commit() - errcode = ' (cdb2api rc %d)' % cdb2.ERROR_CODE['DUPLICATE'] + errcode = " (cdb2api rc %d)" % cdb2.ERROR_CODE["DUPLICATE"] assert errcode in str(exc_info.value) cursor.execute("insert into simple(key, val) values(null, 2)") with pytest.raises(NonNullConstraintError) as exc_info: conn.commit() - errcode = ' (cdb2api rc %d)' % cdb2.ERROR_CODE['NULL_CONSTRAINT'] + errcode = " (cdb2api rc %d)" % cdb2.ERROR_CODE["NULL_CONSTRAINT"] assert errcode in str(exc_info.value) cursor.execute("insert into simple(key, val) values(1, 2)") conn.commit() cursor.execute("selectv * from simple") - connect('mattdb', 'dev', autocommit=True).cursor().execute( - "update simple set key=2") + connect("mattdb", "dev", autocommit=True).cursor().execute( + "update simple set key=2" + ) with pytest.raises(IntegrityError) as exc_info: conn.commit() - errcode = ' (cdb2api rc %d)' % cdb2.ERROR_CODE['CONSTRAINTS'] + errcode = " (cdb2api rc %d)" % cdb2.ERROR_CODE["CONSTRAINTS"] assert errcode in str(exc_info.value) cursor.execute("insert into child(key) values(1)") with pytest.raises(ForeignKeyConstraintError) as exc_info: conn.commit() - errcode = ' (cdb2api rc %d)' % cdb2.ERROR_CODE['FKEY_VIOLATION'] + errcode = " (cdb2api rc %d)" % cdb2.ERROR_CODE["FKEY_VIOLATION"] assert errcode in str(exc_info.value) def test_implicitly_closing_old_cursor_when_opening_a_new_one(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor1 = conn.cursor() cursor2 = conn.cursor() with pytest.raises(InterfaceError): @@ -209,7 +223,7 @@ def test_implicitly_closing_old_cursor_when_opening_a_new_one(): def test_commit_after_cursor_close(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("insert into simple(key, val) values(1, 2)") conn.commit() @@ -223,40 +237,78 @@ def test_commit_after_cursor_close(): def test_implicit_rollback_on_connection_close(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("insert into simple(key, val) values(1, 2)") conn.commit() cursor.execute("insert into simple(key, val) values(3, 4)") - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() rows = list(cursor.execute("select key, val from simple order by key")) assert rows == [[1, 2]] def test_extra_percent_arg(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() with pytest.raises(InterfaceError): - cursor.execute("insert into simple(key, val) values(%(k)s, %(v)s)", - dict(k=3)) + cursor.execute("insert into simple(key, val) values(%(k)s, %(v)s)", dict(k=3)) -def test_unescaped_percent(): - conn = connect('mattdb', 'dev') +def test_binding_no_parameters(): + conn = connect("mattdb", "dev") cursor = conn.cursor() - cursor.execute("select 1%%2" ) # Should work + + rows = list(cursor.execute("select 1 %% 10")) + assert rows == [[1]] + + rows = list(cursor.execute("select 2 %% 10", {})) + assert rows == [[2]] + + rows = list(cursor.execute("select 3 % 10", ())) + assert rows == [[3]] + + rows = list(cursor.execute("select 4 % 10", [])) + assert rows == [[4]] + + +def test_incorrect_escaping_of_percent_signs(): + conn = connect("mattdb", "dev") + cursor = conn.cursor() + + with pytest.raises(InterfaceError): + cursor.execute("select 1 % 10") + with pytest.raises(InterfaceError): - cursor.execute("select 1%2") + cursor.execute("select 2 % 10", {}) + + with pytest.raises(ProgrammingError): + cursor.execute("select 3 %% 10", ()) + + with pytest.raises(ProgrammingError): + cursor.execute("select 4 %% 10", []) + + +def test_binding_different_sequence_types(): + conn = connect("mattdb", "dev") + cursor = conn.cursor() + cursor.execute("select ?, ?", [1, 2]) + assert cursor.fetchall() == [[1, 2]] + + cursor.execute("select ?, ?", (1, 2)) + assert cursor.fetchall() == [[1, 2]] + + cursor.execute("select ?, ?", "hi") + assert cursor.fetchall() == [["h", "i"]] def test_reading_and_writing_datetimes(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() - ts_obj = Timestamp(2015, 1, 2, 3, 4, 5, 123000, pytz.UTC) - ts_str_in = '2015-01-02T03:04:05.12345' - ts_str_out = '2015-01-02T030405.123 UTC' + ts_obj = Timestamp(2015, 1, 2, 3, 4, 5, 123000, datetime.timezone.utc) + ts_str_in = "2015-01-02T03:04:05.12345" + ts_str_out = "2015-01-02T030405.123 UTC" cursor.execute("select cast(%(x)s as date)", dict(x=ts_str_in)) assert cursor.fetchall() == [[ts_obj]] @@ -266,11 +318,11 @@ def test_reading_and_writing_datetimes(): def test_reading_and_writing_datetimeus(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() - ts_obj = TimestampUs(2015, 1, 2, 3, 4, 5, 123456, pytz.UTC) - ts_str_in = '2015-01-02T03:04:05.123456' - ts_str_out = '2015-01-02T030405.123456 UTC' + ts_obj = TimestampUs(2015, 1, 2, 3, 4, 5, 123456, datetime.timezone.utc) + ts_str_in = "2015-01-02T03:04:05.123456" + ts_str_out = "2015-01-02T030405.123456 UTC" cursor.execute("select cast(%(x)s as date)", dict(x=ts_str_in)) assert cursor.fetchall() == [[ts_obj]] @@ -280,37 +332,49 @@ def test_reading_and_writing_datetimeus(): def test_inserting_one_row_with_all_datatypes_without_parameters(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() - cursor.execute("insert into all_datatypes(" + ', '.join(COLUMN_LIST) + ")" - " values(1, 2, 3, 4, 5, .5, .25, x'01', x'0102030405'," - " 'hello', 'goodbye', x'01020304050607'," - " cast(1234567890.2345 as datetime), 'hello world')") + cursor.execute( + "insert into all_datatypes(" + ", ".join(COLUMN_LIST) + ")" + " values(1, 2, 3, 4, 5, .5, .25, x'01', x'0102030405'," + " 'hello', 'goodbye', x'01020304050607'," + " cast(1234567890.2345 as datetime), 'hello world')" + ) conn.commit() assert cursor.rowcount == 1 - cursor.execute("select " + ', '.join(COLUMN_LIST) + " from all_datatypes") + cursor.execute("select " + ", ".join(COLUMN_LIST) + " from all_datatypes") for i in range(len(COLUMN_LIST)): assert cursor.description[i][0] == COLUMN_LIST[i] for type_object in (STRING, BINARY, NUMBER, DATETIME): - assert ((type_object == cursor.description[i][1]) - == (type_object == COLUMN_TYPE[i])) + assert (type_object == cursor.description[i][1]) == ( + type_object == COLUMN_TYPE[i] + ) assert cursor.description[i][2:] == (None, None, None, None, None) row = cursor.fetchone() - assert row == [1, 2, 3, 4, 5, 0.5, 0.25, - Binary('\x01'), Binary('\x01\x02\x03\x04\x05'), - 'hello', 'goodbye', - Binary('\x01\x02\x03\x04\x05\x06\x07'), - pytz.timezone("America/New_York").localize( - Datetime(2009, 2, 13, 18, 31, 30, 234000)), - "hello world"] + assert row == [ + 1, + 2, + 3, + 4, + 5, + 0.5, + 0.25, + Binary("\x01"), + Binary("\x01\x02\x03\x04\x05"), + "hello", + "goodbye", + Binary("\x01\x02\x03\x04\x05\x06\x07"), + Datetime(2009, 2, 13, 18, 31, 30, 234000, gettz("America/New_York")), + "hello world", + ] assert cursor.fetchone() is None def test_all_datatypes_as_parameters(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() params = ( ("short_col", 32767), @@ -318,23 +382,28 @@ def test_all_datatypes_as_parameters(): ("int_col", 2147483647), ("u_int_col", 4294967295), ("longlong_col", 9223372036854775807), - ("float_col", .125), - ("double_col", 2.**65), - ("byte_col", Binary(b'\x00')), - ("byte_array_col", Binary(b'\x02\x01\x00\x01\x02')), - ("cstring_col", 'HELLO'), - ("pstring_col", 'GOODBYE'), - ("blob_col", Binary('')), - ("datetime_col", pytz.timezone("America/New_York").localize( - Datetime(2009, 2, 13, 18, 31, 30, 234000))), - ("vutf8_col", "foo" * 50) + ("float_col", 0.125), + ("double_col", 2.0**65), + ("byte_col", Binary(b"\x00")), + ("byte_array_col", Binary(b"\x02\x01\x00\x01\x02")), + ("cstring_col", "HELLO"), + ("pstring_col", "GOODBYE"), + ("blob_col", Binary("")), + ( + "datetime_col", + Datetime(2009, 2, 13, 18, 31, 30, 234000, gettz("America/New_York")) + ), + ("vutf8_col", "foo" * 50), + ) + cursor.execute( + "insert into all_datatypes(" + ", ".join(COLUMN_LIST) + ")" + " values(%(short_col)s, %(u_short_col)s, %(int_col)s," + " %(u_int_col)s, %(longlong_col)s, %(float_col)s," + " %(double_col)s, %(byte_col)s, %(byte_array_col)s," + " %(cstring_col)s, %(pstring_col)s, %(blob_col)s," + " %(datetime_col)s, %(vutf8_col)s)", + dict(params), ) - cursor.execute("insert into all_datatypes(" + ', '.join(COLUMN_LIST) + ")" - " values(%(short_col)s, %(u_short_col)s, %(int_col)s," - " %(u_int_col)s, %(longlong_col)s, %(float_col)s," - " %(double_col)s, %(byte_col)s, %(byte_array_col)s," - " %(cstring_col)s, %(pstring_col)s, %(blob_col)s," - " %(datetime_col)s, %(vutf8_col)s)", dict(params)) conn.commit() @@ -343,9 +412,25 @@ def test_all_datatypes_as_parameters(): assert row == list(v for k, v in params) assert cursor.fetchone() is None + cursor.execute("delete from all_datatypes") + conn.commit() + + cursor.execute( + "insert into all_datatypes(" + ", ".join(COLUMN_LIST) + ")" + " values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + [v for _, v in params], + ) + + conn.commit() + + cursor.execute("select * from all_datatypes") + row2 = cursor.fetchone() + assert row2 == row + assert cursor.fetchone() is None + def test_naive_datetime_as_parameter(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() params = ( ("short_col", 32767), @@ -353,66 +438,97 @@ def test_naive_datetime_as_parameter(): ("int_col", 2147483647), ("u_int_col", 4294967295), ("longlong_col", 9223372036854775807), - ("float_col", .125), - ("double_col", 2.**65), - ("byte_col", Binary(b'\x00')), - ("byte_array_col", Binary(b'\x02\x01\x00\x01\x02')), - ("cstring_col", 'HELLO'), - ("pstring_col", 'GOODBYE'), - ("blob_col", Binary('')), + ("float_col", 0.125), + ("double_col", 2.0**65), + ("byte_col", Binary(b"\x00")), + ("byte_array_col", Binary(b"\x02\x01\x00\x01\x02")), + ("cstring_col", "HELLO"), + ("pstring_col", "GOODBYE"), + ("blob_col", Binary("")), ("datetime_col", Datetime(2009, 2, 13, 18, 31, 30, 234000)), - ("vutf8_col", "foo" * 50) + ("vutf8_col", "foo" * 50), ) - cursor.execute("insert into all_datatypes(" + ', '.join(COLUMN_LIST) + ")" - " values(%(short_col)s, %(u_short_col)s, %(int_col)s," - " %(u_int_col)s, %(longlong_col)s, %(float_col)s," - " %(double_col)s, %(byte_col)s, %(byte_array_col)s," - " %(cstring_col)s, %(pstring_col)s, %(blob_col)s," - " %(datetime_col)s, %(vutf8_col)s)", dict(params)) + cursor.execute( + "insert into all_datatypes(" + ", ".join(COLUMN_LIST) + ")" + " values(%(short_col)s, %(u_short_col)s, %(int_col)s," + " %(u_int_col)s, %(longlong_col)s, %(float_col)s," + " %(double_col)s, %(byte_col)s, %(byte_array_col)s," + " %(cstring_col)s, %(pstring_col)s, %(blob_col)s," + " %(datetime_col)s, %(vutf8_col)s)", + dict(params), + ) cursor.connection.commit() cursor.execute("select datetime_col from all_datatypes") row = cursor.fetchone() - assert row == [Datetime(2009, 2, 13, 18, 31, 30, 234000, pytz.UTC)] + assert row == [Datetime(2009, 2, 13, 18, 31, 30, 234000, datetime.timezone.utc)] + assert cursor.fetchone() is None + + cursor.execute("delete from all_datatypes") + conn.commit() + + cursor.execute( + "insert into all_datatypes(" + ", ".join(COLUMN_LIST) + ")" + " values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + list(v for k, v in params), + ) + + conn.commit() + + cursor.execute("select datetime_col from all_datatypes") + row2 = cursor.fetchone() + assert row2 == row assert cursor.fetchone() is None def test_datetime_with_non_olson_tzname(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() - tz = pytz.timezone('America/New_York') + tz = gettz("America/New_York") dt = datetime.datetime(2016, 11, 6, 1, 30, 0, 123000) - est_dt = tz.localize(dt, is_dst=False) - edt_dt = tz.localize(dt, is_dst=True) - assert est_dt.tzname() == 'EST' - assert edt_dt.tzname() == 'EDT' - params = {'est_dt': est_dt, 'edt_dt': edt_dt} + est_dt = dt.replace(tzinfo=tz, fold=1) + edt_dt = dt.replace(tzinfo=tz, fold=0) + assert est_dt.tzname() == "EST" + assert edt_dt.tzname() == "EDT" + params = {"est_dt": est_dt, "edt_dt": edt_dt} row = cursor.execute("select @est_dt, @edt_dt", params).fetchone() - assert row[0].tzname() == 'UTC' - assert row[0] == est_dt - assert row[1].tzname() == 'UTC' - assert row[1] == edt_dt + assert row[0].tzname() == "UTC" + assert row[0] == est_dt.astimezone(datetime.timezone.utc) + assert row[1].tzname() == "UTC" + assert row[1] == edt_dt.astimezone(datetime.timezone.utc) + + row = cursor.execute("select ?, ?", [est_dt, edt_dt]).fetchone() + assert row[0].tzname() == "UTC" + assert row[0] == est_dt.astimezone(datetime.timezone.utc) + assert row[1].tzname() == "UTC" + assert row[1] == edt_dt.astimezone(datetime.timezone.utc) def test_rounding_datetime_to_nearest_millisecond(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() - curr_microsecond = Datetime(2016, 2, 28, 23, 59, 59, 999499, pytz.UTC) - prev_millisecond = Datetime(2016, 2, 28, 23, 59, 59, 999000, pytz.UTC) - next_millisecond = Datetime(2016, 2, 29, 0, 0, 0, 0, pytz.UTC) + curr_microsecond = Datetime(2016, 2, 28, 23, 59, 59, 999499, datetime.timezone.utc) + prev_millisecond = Datetime(2016, 2, 28, 23, 59, 59, 999000, datetime.timezone.utc) + next_millisecond = Datetime(2016, 2, 29, 0, 0, 0, 0, datetime.timezone.utc) + + cursor.execute("select @date", {"date": curr_microsecond}) + assert cursor.fetchall() == [[prev_millisecond]] - cursor.execute("select @date", {'date': curr_microsecond}) + cursor.execute("select ?", [curr_microsecond]) assert cursor.fetchall() == [[prev_millisecond]] curr_microsecond += datetime.timedelta(microseconds=1) - cursor.execute("select @date", {'date': curr_microsecond}) + cursor.execute("select @date", {"date": curr_microsecond}) + assert cursor.fetchall() == [[next_millisecond]] + + cursor.execute("select ?", [curr_microsecond]) assert cursor.fetchall() == [[next_millisecond]] def test_cursor_description(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() assert cursor.description is None @@ -426,12 +542,14 @@ def test_cursor_description(): cursor.execute("select '1' as foo, cast(1 as datetime) bar") assert cursor.description == ( ("foo", STRING, None, None, None, None, None), - ("bar", DATETIME, None, None, None, None, None)) + ("bar", DATETIME, None, None, None, None, None), + ) cursor.connection.commit() assert cursor.description == ( ("foo", STRING, None, None, None, None, None), - ("bar", DATETIME, None, None, None, None, None)) + ("bar", DATETIME, None, None, None, None, None), + ) cursor.execute("insert into simple(key, val) values(3, 4)") assert cursor.description is None @@ -442,12 +560,14 @@ def test_cursor_description(): cursor.execute("select key, val from simple") assert cursor.description == ( ("key", NUMBER, None, None, None, None, None), - ("val", NUMBER, None, None, None, None, None)) + ("val", NUMBER, None, None, None, None, None), + ) cursor.connection.rollback() assert cursor.description == ( ("key", NUMBER, None, None, None, None, None), - ("val", NUMBER, None, None, None, None, None)) + ("val", NUMBER, None, None, None, None, None), + ) with pytest.raises(ProgrammingError): cursor.execute("select") @@ -455,21 +575,24 @@ def test_cursor_description(): def test_binding_number_that_overflows_long_long(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() with pytest.raises(DataError): cursor.execute("select @i", dict(i=2**64 + 1)) + with pytest.raises(DataError): + cursor.execute("select ?", [2**64 + 1]) + def test_retrieving_null(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("select null, null") assert cursor.fetchall() == [[None, None]] def test_retrieving_interval(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("select cast(0 as datetime) - cast(0 as datetime)") with pytest.raises(NotSupportedError): @@ -477,7 +600,7 @@ def test_retrieving_interval(): def test_syntax_error(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("foo") with pytest.raises(ProgrammingError): @@ -485,21 +608,21 @@ def test_syntax_error(): def test_errors_being_swallowed_during_cursor_close(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("foo") cursor.close() def test_errors_being_swallowed_during_connection_close(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("foo") conn.close() def test_public_connection_methods_after_close(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") conn.close() with pytest.raises(InterfaceError): conn.close() @@ -517,45 +640,45 @@ def test_public_connection_methods_after_close(): def test_misusing_cursor_objects(): with pytest.raises(InterfaceError): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute(" BEGIN TRANSACTION ") with pytest.raises(InterfaceError): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("commit") with pytest.raises(InterfaceError): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute(" rollback ") with pytest.raises(InterfaceError): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.fetchone() with pytest.raises(InterfaceError): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.fetchmany() with pytest.raises(InterfaceError): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.fetchall() def test_noop_methods(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.setinputsizes([1, 2, 3]) cursor.setoutputsize(42) def test_fetchmany(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("select 1 UNION select 2 UNION select 3 order by 1") @@ -577,18 +700,20 @@ def test_fetchmany(): def test_consuming_result_sets_automatically(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("select 1 UNION select 2 UNION select 3 order by 1") cursor.execute("select 1 UNION select 2 UNION select 3 order by 1") def test_inserting_non_utf8_string(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() - cursor.execute("insert into strings values(cast(%(x)s as text), %(y)s)", - dict(x=b'\x68\xeb\x6c\x6c\x6f', y=b'\x68\xeb\x6c\x6c\x6f')) + cursor.execute( + "insert into strings values(cast(%(x)s as text), %(y)s)", + dict(x=b"\x68\xeb\x6c\x6c\x6f", y=b"\x68\xeb\x6c\x6c\x6f"), + ) conn.commit() with pytest.raises(DataError): @@ -596,11 +721,11 @@ def test_inserting_non_utf8_string(): rows = list(cursor) rows = list(cursor.execute("select cast(foo as blob), bar from strings")) - assert rows == [[b'\x68\xeb\x6c\x6c\x6f', b'\x68\xeb\x6c\x6c\x6f']] + assert rows == [[b"\x68\xeb\x6c\x6c\x6f", b"\x68\xeb\x6c\x6c\x6f"]] def test_cursor_connection_attribute_keeps_connection_alive(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() del conn cursor.execute("insert into simple(key, val) values(1, 2)") @@ -612,37 +737,37 @@ def test_cursor_connection_attribute_keeps_connection_alive(): def test_exceptions_containing_unicode_error_messages(): - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() with pytest.raises(ProgrammingError): try: cursor.execute("select") except ProgrammingError as exc: - assert isinstance(exc.args[0], six.text_type) + assert isinstance(exc.args[0], str) raise -def throw_on(expected_stmt, stmt, parameters=None): +def throw_on(expected_stmt, stmt, parameters=None, column_types=None): if stmt == expected_stmt: - raise cdb2.Error(42, 'Not supported error') + raise cdb2.Error(42, "Not supported error") -@patch('comdb2.cdb2.Handle') +@patch("comdb2.cdb2.Handle") def test_begin_throws_error(handle): - handle.return_value.execute.side_effect = partial(throw_on, 'begin') + handle.return_value.execute.side_effect = partial(throw_on, "begin") - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() with pytest.raises(OperationalError): cursor.execute("insert into simple(key, val) values(1, 2)") -@patch('comdb2.cdb2.Handle') +@patch("comdb2.cdb2.Handle") def test_commit_throws_error(handle): - handle.return_value.execute.side_effect = partial(throw_on, 'commit') + handle.return_value.execute.side_effect = partial(throw_on, "commit") - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("insert into simple(key, val) values(1, 2)") @@ -650,21 +775,36 @@ def test_commit_throws_error(handle): conn.commit() -@patch('comdb2.cdb2.Handle') +@patch("comdb2.cdb2.Handle") def test_get_effect_throws_error(handle): def raise_not_supported_error(): - raise cdb2.Error(42, 'Not supported error') + raise cdb2.Error(42, "Not supported error") handle.return_value.get_effects.side_effect = raise_not_supported_error - conn = connect('mattdb', 'dev') + conn = connect("mattdb", "dev") cursor = conn.cursor() cursor.execute("insert into simple(key, val) values(1, 2)") conn.commit() +@patch("comdb2.cdb2.Handle") +def test_error_wraps_underlying(handle): + handle.return_value.execute.side_effect = partial(throw_on, "begin") + + conn = connect("mattdb", "dev") + cursor = conn.cursor() + + with pytest.raises(OperationalError) as exc_info: + cursor.execute("insert into simple(key, val) values(1, 2)") + + assert isinstance(exc_info.value.__cause__, cdb2.Error) + underlying = exc_info.value.__cause__ + assert underlying.error_code == 42 + + def test_autocommit_handles(): - conn = connect('mattdb', 'dev', autocommit=True) + conn = connect("mattdb", "dev", autocommit=True) cursor = conn.cursor() # Explicit transactions must work @@ -712,24 +852,24 @@ def test_autocommit_handles(): def test_row_factories(): query = "select 1 as 'a', 2 as 'b' union select 3, 4 order by a" - hndl = connect('mattdb', 'dev') + hndl = connect("mattdb", "dev") assert list(hndl.cursor().execute(query)) == [[1, 2], [3, 4]] hndl.row_factory = dict_row_factory assert hndl.row_factory == dict_row_factory rows = list(hndl.cursor().execute(query)) - assert rows == [{'a': 1, 'b': 2}, {'a': 3, 'b': 4}] + assert rows == [{"a": 1, "b": 2}, {"a": 3, "b": 4}] hndl.row_factory = namedtuple_row_factory assert hndl.row_factory == namedtuple_row_factory rows = [r._asdict() for r in hndl.cursor().execute(query)] - assert rows == [{'a': 1, 'b': 2}, {'a': 3, 'b': 4}] + assert rows == [{"a": 1, "b": 2}, {"a": 3, "b": 4}] def test_row_factories_with_dup_col_names(): query = "select 1 as 'a', 2 as 'a', 3 as 'b', 4 as 'b', 5 as 'c'" - hndl = connect('mattdb', 'dev') + hndl = connect("mattdb", "dev") assert list(hndl.cursor().execute(query)) == [[1, 2, 3, 4, 5]] @@ -744,7 +884,7 @@ def test_row_factories_with_dup_col_names(): def test_row_factories_with_reserved_word_col_names(): query = "select 1 as 'def'" - hndl = connect('mattdb', 'dev') + hndl = connect("mattdb", "dev") assert list(hndl.cursor().execute(query)) == [[1]] @@ -753,21 +893,21 @@ def test_row_factories_with_reserved_word_col_names(): hndl.cursor().execute(query) hndl.row_factory = dict_row_factory - assert list(hndl.cursor().execute(query)) == [{'def': 1}] + assert list(hndl.cursor().execute(query)) == [{"def": 1}] def test_reusing_handle_after_unicode_decode_error(): - hndl = connect('mattdb', 'dev') + hndl = connect("mattdb", "dev") cursor = hndl.cursor() with pytest.raises(DataError): cursor.execute("select cast(X'C3' as text)").fetchall() row = cursor.execute("select cast(X'C3A4' as text)").fetchone() - assert row == ['\xE4'] + assert row == ["\xe4"] def test_unicode_column_decode_exception(): query = "select cast(X'C3A4' as text) as a, cast(X'C3' as text) as b" - hndl = connect('mattdb', 'dev') + hndl = connect("mattdb", "dev") cursor = hndl.cursor() cursor.execute(query) @@ -783,7 +923,7 @@ def test_unicode_column_decode_exception(): def test_date_column_decode_exception(): query = "select cast('0000-01-01 UTC' as date) as date" - hndl = connect('mattdb', 'dev') + hndl = connect("mattdb", "dev") cursor = hndl.cursor() cursor.execute("SET TIMEZONE America/New_York") @@ -793,29 +933,34 @@ def test_date_column_decode_exception(): exc_str = str(exc_info.value) assert "Failed to decode CDB2_DATETIME column 0 ('date'):" in exc_str - assert " out of range" in exc_str + # Check for both old and new error message formats + # Python < 3.14: "year 0 is out of range" + # Python >= 3.14: "year must be in MINYEAR..MAXYEAR, not 0" + # See: https://github.com/python/cpython/commit/3e222e3a15959690a41847a1177ac424427815e5#diff-3ee250e3806e884518fd872e9148baf532de6ec54c1cdb4e7679fbb2869d9c47 + assert (" out of range" in exc_str or "year must be in" in exc_str) def test_unsupported_column_decode_exception(): query = "select now() - now() as delta" - hndl = connect('mattdb', 'dev') + hndl = connect("mattdb", "dev") cursor = hndl.cursor() cursor.execute(query) with pytest.raises(NotSupportedError) as exc_info: cursor.fetchall() - errmsg = ("Failed to decode CDB2_INTERVALDS column 0 ('delta'):" - " Unsupported column type") + errmsg = ( + "Failed to decode CDB2_INTERVALDS column 0 ('delta'): Unsupported column type" + ) assert errmsg in str(exc_info.value) def test_datetimeus(): query = "select %(date)s + cast(30 as days)" - hndl = connect('mattdb', 'dev') + hndl = connect("mattdb", "dev") cursor = hndl.cursor() - sent = DatetimeUs(2017, 8, 16, 19, 32, 2, 825022, tzinfo=pytz.UTC) + sent = DatetimeUs(2017, 8, 16, 19, 32, 2, 825022, tzinfo=datetime.timezone.utc) cursor.execute(query, dict(date=sent)) rcvd = cursor.fetchall()[0][0] @@ -823,7 +968,7 @@ def test_datetimeus(): def test_interface_error_reading_result_set_after_commits(): - hndl = connect('mattdb', 'dev') + hndl = connect("mattdb", "dev") cursor = hndl.cursor().execute("delete from simple where 1=1") assert cursor.description is None hndl.commit() @@ -831,7 +976,7 @@ def test_interface_error_reading_result_set_after_commits(): cursor.fetchall() assert "No result set exists" in str(exc_info.value) - hndl = connect('mattdb', 'dev', autocommit=True) + hndl = connect("mattdb", "dev", autocommit=True) cursor = hndl.cursor().execute("delete from simple where 1=1") assert cursor.description is None with pytest.raises(InterfaceError) as exc_info: @@ -839,20 +984,139 @@ def test_interface_error_reading_result_set_after_commits(): assert "No result set exists" in str(exc_info.value) -@pytest.mark.parametrize("statement, operation", [ - ("select 1", "select"), - (" \n\tselect 1", "select"), - (" /* */ select 1", "select"), - ("\n--\n\t\tselect 1", "select"), - ("insert/**/1", "insert"), - ("-- select 1\n-- Insert 2\nUpdate 3", "update"), - ("/*select 1*/\n/*Insert 2*/\nUpdate 3", "update"), - ("/* foo\nbar\nbaz\n-- INSERT*/select foo", "select"), - ("/* foo\nbar\nbaz\n-- INSERT*/--select\ninsert foo", "insert"), - ("-- /*\n*/ foo", None), - ("--", None), - ("*/", None), - ("", None), -]) +@pytest.mark.parametrize( + "statement, operation", + [ + ("select 1", "select"), + (" \n\tselect 1", "select"), + (" /* */ select 1", "select"), + ("\n--\n\t\tselect 1", "select"), + ("insert/**/1", "insert"), + ("-- select 1\n-- Insert 2\nUpdate 3", "update"), + ("/*select 1*/\n/*Insert 2*/\nUpdate 3", "update"), + ("/* foo\nbar\nbaz\n-- INSERT*/select foo", "select"), + ("/* foo\nbar\nbaz\n-- INSERT*/--select\ninsert foo", "insert"), + ("-- /*\n*/ foo", None), + ("--", None), + ("*/", None), + ("", None), + ], +) def test_finding_operation(statement, operation): assert _sql_operation(statement) == operation + + +@pytest.mark.parametrize( + "values", + [ + [5, 10], + ("hello", "hi"), + [0.25, 0.35, 0.25], + (b"123", b"456"), + ], +) +def test_parameter_binding_arrays(values): + # GIVEN + conn = connect("mattdb", "dev") + cursor = conn.cursor() + + # WHEN + cursor.execute("select * from carray(%(values)s)", dict(values=values)) + results = cursor.fetchall() + + # THEN + assert results == [[v] for v in values] + + # WHEN + cursor.execute("select * from carray(?)", [values]) + results = cursor.fetchall() + + # THEN + assert results == [[v] for v in values] + conn.close() + + +@pytest.mark.parametrize( + "values,exc_msg", + [ + ( + [], + "Can't bind list value [] for parameter {}: " + + "ValueError: empty lists cannot be bound", + ), + ( + (), + "Can't bind tuple value () for parameter {}: " + + "ValueError: empty tuples cannot be bound", + ), + ( + [1, "hello"], + "Can't bind list value [1, 'hello'] for parameter {}: " + + "ValueError: all list elements must be the same type", + ), + ( + (1j,), + "Can't bind tuple value (1j,) for parameter {}: " + + "ValueError: Cannot bind a tuple of complex", + ), + ( + ((1, 2, 3),), + "Can't bind tuple value ((1, 2, 3),) for parameter {}: " + + "ValueError: Cannot bind a tuple of tuple", + ), + ], +) +def test_parameter_binding_invalid_arrays(values, exc_msg): + # GIVEN + conn = connect("mattdb", "dev") + cursor = conn.cursor() + + # WHEN/THEN + with pytest.raises(DataError, match=re.escape(exc_msg.format("'values'"))): + cursor.execute("select * from carray(%(values)s)", dict(values=values)) + + # WHEN/THEN + with pytest.raises(DataError, match=re.escape(exc_msg.format("1"))): + cursor.execute("select * from carray(?)", [values]) + + +def test_specifying_column_types(): + # GIVEN + conn = connect("mattdb", "dev") + cursor = conn.cursor() + column_types = [ + cdb2.ColumnType.INT, + cdb2.ColumnType.REAL, + cdb2.ColumnType.TEXT, + cdb2.ColumnType.BLOB, + cdb2.ColumnType.DATETIME, + cdb2.ColumnType.DATETIMEUS, + ] + sql = "select 0, 0, 0, 0, 0, 0" + + # WHEN + cursor.execute(sql, column_types=column_types) + row = cursor.fetchone() + + # THEN + assert row is not None + assert type(row[0]) is int + assert type(row[1]) is float + assert type(row[2]) is str + assert type(row[3]) is bytes + assert type(row[4]) is datetime.datetime + assert type(row[5]) is cdb2.DatetimeUs + + +def test_providing_empty_column_types_array(): + # GIVEN + hndl = cdb2.Handle("mattdb", "dev") + column_types = [] + sql = "select 0" + + # WHEN + (row,) = hndl.execute(sql, column_types=column_types) + + # THEN + assert row is not None + assert type(row[0]) is int