diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cac01d4d..6632f74a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -104,6 +104,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), before; users should call `cursor.setinputsizes()` to work around this. ### Fixed +- **GH-855:** Bulk copy and internal async queries load the Rust extension directly + when upgrading from 1.15.0 removes `mssql_py_core/__init__.py`. Existing affected + installations can restore the initializer with + `python -m pip install --force-reinstall --no-deps mssql-python-rs==0.3.0`, + followed by restarting Python. - **GH-754:** Pooled connections are now rolled back and restored to autocommit mode before being parked. This prevents an empty transaction from remaining visible on an idle SQL Server session after `Connection.close()`. Abandoned diff --git a/mssql_python/async_query/_native.py b/mssql_python/async_query/_native.py index 8cd282335..9cbe7f389 100644 --- a/mssql_python/async_query/_native.py +++ b/mssql_python/async_query/_native.py @@ -7,7 +7,8 @@ def load_py_core() -> ModuleType: """Load the PyO3 extension that owns asynchronous TDS operations.""" try: - py_core = import_module("mssql_py_core") + # Upgrading from a bundled core can remove the package's __init__.py. + py_core = import_module("mssql_py_core.mssql_py_core") except ImportError as exc: raise ImportError( "Async query support requires the mssql-python-rs dependency " diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index d4b85e56b..cd9447f55 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -16,6 +16,7 @@ import uuid import datetime import warnings +from importlib import import_module from typing import List, Mapping, Union, Any, Optional, Tuple, Sequence, TYPE_CHECKING, Iterable from mssql_python.constants import ConstantsDDBC as ddbc_sql_const, SQLTypes from mssql_python.helpers import check_error, connstr_to_pycore_params @@ -3447,9 +3448,10 @@ def _looks_like_arrow_source(self, data) -> bool: @staticmethod def _bulkcopy_core_and_validate(table_name, batch_size, timeout): """Import the native core and validate the args shared by ``bulkcopy`` - and ``bulkcopy_arrow``. Returns the imported ``mssql_py_core`` module.""" + and ``bulkcopy_arrow``. Returns the native ``mssql_py_core`` extension.""" try: - import mssql_py_core + # Upgrading from a bundled core can remove the package's __init__.py. + mssql_py_core = import_module("mssql_py_core.mssql_py_core") except ImportError as exc: logger.error("bulkcopy: Failed to import mssql_py_core module") raise ImportError( diff --git a/tests/AsyncTest/test_001_async_query_native.py b/tests/AsyncTest/test_001_async_query_native.py index 9685bbecc..7cc34bd15 100644 --- a/tests/AsyncTest/test_001_async_query_native.py +++ b/tests/AsyncTest/test_001_async_query_native.py @@ -1,3 +1,5 @@ +import sys +from importlib.machinery import ModuleSpec from types import ModuleType import pytest @@ -19,15 +21,29 @@ def test_async_entry_points_are_internal(): def test_load_py_core_uses_direct_native_dependency(): - pytest.importorskip("mssql_py_core", exc_type=ImportError) + expected = pytest.importorskip("mssql_py_core.mssql_py_core", exc_type=ImportError) py_core = native.load_py_core() - assert py_core.__name__ == "mssql_py_core" + assert py_core is expected assert py_core.PyAsyncConnection.__name__ == "PyAsyncConnection" assert py_core.PyAsyncCursor.__name__ == "PyAsyncCursor" assert getattr(py_core, "PyAsyncCursor").__name__ == "PyAsyncCursor" +def test_load_py_core_without_package_initializer(monkeypatch): + package = ModuleType("mssql_py_core") + package.__path__ = [] + package.__spec__ = ModuleSpec("mssql_py_core", loader=None, is_package=True) + py_core = ModuleType("mssql_py_core.mssql_py_core") + py_core.PyAsyncConnection = type("PyAsyncConnection", (), {}) + py_core.PyAsyncCursor = type("PyAsyncCursor", (), {}) + monkeypatch.setitem(sys.modules, "mssql_py_core", package) + monkeypatch.setitem(sys.modules, "mssql_py_core.mssql_py_core", py_core) + + assert not hasattr(package, "PyAsyncConnection") + assert native.load_py_core() is py_core + + def test_load_py_core_reports_missing_dependency(monkeypatch): def raise_import_error(name): raise ImportError(f"No module named {name}") @@ -41,7 +57,7 @@ def raise_import_error(name): def test_load_py_core_reports_missing_async_types(monkeypatch): - py_core = ModuleType("mssql_py_core") + py_core = ModuleType("mssql_py_core.mssql_py_core") monkeypatch.setattr(native, "import_module", lambda name: py_core) with pytest.raises(ImportError, match="PyAsyncConnection, PyAsyncCursor"): diff --git a/tests/test_008_auth.py b/tests/test_008_auth.py index 5c160b2a4..3bee95695 100644 --- a/tests/test_008_auth.py +++ b/tests/test_008_auth.py @@ -877,7 +877,7 @@ def fake_get_raw_token(auth_type, credential_kwargs=None): mock_pycore_module.PyCoreConnection = lambda ctx, **kwargs: mock_pycore_conn with ( - patch.dict("sys.modules", {"mssql_py_core": mock_pycore_module}), + patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": mock_pycore_module}), patch("mssql_python.auth.AADAuth.get_raw_token", side_effect=fake_get_raw_token), ): cursor.bulkcopy("dbo.test_table", [(1, "row")], timeout=10) diff --git a/tests/test_020_bulkcopy_auth_cleanup.py b/tests/test_020_bulkcopy_auth_cleanup.py index b033baac0..b85347cc5 100644 --- a/tests/test_020_bulkcopy_auth_cleanup.py +++ b/tests/test_020_bulkcopy_auth_cleanup.py @@ -10,7 +10,10 @@ """ import secrets +import sys from enum import IntEnum +from importlib.machinery import ModuleSpec +from types import ModuleType from unittest.mock import MagicMock, patch import pytest @@ -38,6 +41,42 @@ def _make_cursor(connection_str, auth_type): return cursor +@pytest.mark.parametrize("method", ["bulkcopy", "bulkcopy_arrow"]) +def test_bulkcopy_without_package_initializer(monkeypatch, method): + package = ModuleType("mssql_py_core") + package.__path__ = [] + package.__spec__ = ModuleSpec("mssql_py_core", loader=None, is_package=True) + native = ModuleType("mssql_py_core.mssql_py_core") + native.PyCoreConnection = MagicMock() + connection = native.PyCoreConnection.return_value + core_cursor = connection.cursor.return_value + expected = {"rows_copied": 1, "batch_count": 1, "elapsed_time": 0.1} + getattr(core_cursor, method).return_value = expected + monkeypatch.setitem(sys.modules, "mssql_py_core", package) + monkeypatch.setitem(sys.modules, "mssql_py_core.mssql_py_core", native) + cursor = _make_cursor("Server=example.test;Database=testdb;Trusted_Connection=yes", None) + source = [(1,)] if method == "bulkcopy" else object() + + assert not hasattr(package, "PyCoreConnection") + assert getattr(cursor, method)("dbo.test_table", source) == expected + native.PyCoreConnection.assert_called_once() + getattr(core_cursor, method).assert_called_once() + core_cursor.close.assert_called_once() + connection.close.assert_called_once() + + +@pytest.mark.parametrize("method", ["bulkcopy", "bulkcopy_arrow"]) +def test_bulkcopy_reports_missing_native_dependency(monkeypatch, method): + monkeypatch.setitem(sys.modules, "mssql_py_core.mssql_py_core", None) + cursor = _make_cursor("Server=example.test;Database=testdb;Trusted_Connection=yes", None) + source = [(1,)] if method == "bulkcopy" else object() + + with pytest.raises(ImportError, match="Bulk copy requires") as exc_info: + getattr(cursor, method)("dbo.test_table", source) + + assert isinstance(exc_info.value.__cause__, ImportError) + + class TestBulkcopyAuthCleanup: """Verify cursor.bulkcopy strips stale auth fields after token acquisition.""" @@ -71,7 +110,7 @@ def capture_context(ctx, **kwargs): mock_pycore_module.PyCoreConnection = capture_context with ( - patch.dict("sys.modules", {"mssql_py_core": mock_pycore_module}), + patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": mock_pycore_module}), patch("mssql_python.auth.AADAuth.get_raw_token", return_value=SAMPLE_TOKEN), ): cursor.bulkcopy("dbo.test_table", [(1, "row")], timeout=10) @@ -109,7 +148,7 @@ def capture_context(ctx, **kwargs): mock_pycore_module = MagicMock() mock_pycore_module.PyCoreConnection = capture_context - with patch.dict("sys.modules", {"mssql_py_core": mock_pycore_module}): + with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": mock_pycore_module}): cursor.bulkcopy("dbo.test_table", [(1, "row")], timeout=10) assert "access_token" not in captured_context @@ -159,7 +198,7 @@ def capture_context(ctx, **kwargs): mock_pycore_module = MagicMock() mock_pycore_module.PyCoreConnection = capture_context - with patch.dict("sys.modules", {"mssql_py_core": mock_pycore_module}): + with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": mock_pycore_module}): cursor.bulkcopy("dbo.test_table", [(1, "row")], timeout=10) # The credential was consulted for a fresh token. @@ -186,7 +225,7 @@ def test_token_provider_get_token_failure_rewrapped(self, mock_logger): mock_pycore_module = MagicMock() - with patch.dict("sys.modules", {"mssql_py_core": mock_pycore_module}): + with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": mock_pycore_module}): with pytest.raises(OperationalError) as exc_info: cursor.bulkcopy("dbo.test_table", [(1, "row")], timeout=10) @@ -219,7 +258,7 @@ def test_each_bulkcopy_reacquires_fresh_token(self, mock_logger): mock_pycore_module = MagicMock() mock_pycore_module.PyCoreConnection = lambda ctx, **kwargs: mock_pycore_conn - with patch.dict("sys.modules", {"mssql_py_core": mock_pycore_module}): + with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": mock_pycore_module}): cursor.bulkcopy("dbo.t", [(1, "a")], timeout=10) cursor.bulkcopy("dbo.t", [(2, "b")], timeout=10) cursor.bulkcopy("dbo.t", [(3, "c")], timeout=10) @@ -255,7 +294,7 @@ def test_transient_failure_then_recovery(self, mock_logger): mock_pycore_module = MagicMock() mock_pycore_module.PyCoreConnection = lambda ctx, **kwargs: mock_pycore_conn - with patch.dict("sys.modules", {"mssql_py_core": mock_pycore_module}): + with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": mock_pycore_module}): with pytest.raises(OperationalError): cursor.bulkcopy("dbo.t", [(1, "a")], timeout=10) # Cursor still works after the transient failure. @@ -285,7 +324,7 @@ def test_empty_token_from_provider_raises_interface_error(self, mock_logger): mock_pycore_module = MagicMock() - with patch.dict("sys.modules", {"mssql_py_core": mock_pycore_module}): + with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": mock_pycore_module}): with pytest.raises(InterfaceError) as exc_info: cursor.bulkcopy("dbo.test_table", [(1, "row")], timeout=10) @@ -312,7 +351,7 @@ def capture_context(ctx, **kwargs): mock_pycore_module = MagicMock() mock_pycore_module.PyCoreConnection = capture_context - with patch.dict("sys.modules", {"mssql_py_core": mock_pycore_module}): + with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": mock_pycore_module}): cursor.bulkcopy("dbo.test_table", [(1, "row")], timeout=10) return captured_context diff --git a/tests/test_024_bulkcopy_arrow.py b/tests/test_024_bulkcopy_arrow.py index a840853ba..b1deb0a42 100644 --- a/tests/test_024_bulkcopy_arrow.py +++ b/tests/test_024_bulkcopy_arrow.py @@ -25,7 +25,7 @@ # containers without the extension, or pyarrow absent). exc_type=ImportError # so a module that imports but fails its native extension is skipped, not # collected as an error (pytest >= 9.1 default only skips ModuleNotFoundError). -mssql_py_core = pytest.importorskip("mssql_py_core", exc_type=ImportError) +mssql_py_core = pytest.importorskip("mssql_py_core.mssql_py_core", exc_type=ImportError) pa = pytest.importorskip("pyarrow") LIVE_DB = pytest.mark.skipif( @@ -219,7 +219,7 @@ def test_timeout_bool_rejected(self): def test_missing_pycore_raises_importerror(self): cur = _bare_cursor() - with patch.dict("sys.modules", {"mssql_py_core": None}): + with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": None}): with pytest.raises(ImportError, match="mssql_py_core"): cur.bulkcopy_arrow("t", pa.table({"a": [1]})) @@ -372,7 +372,7 @@ def test_success_returns_result_and_forwards_args(self, mock_logger): module, pyc_cursor, _, _ = _mock_pycore() src = pa.table({"a": [1, 2]}) - with patch.dict("sys.modules", {"mssql_py_core": module}): + with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": module}): result = cur.bulkcopy_arrow( "dbo.t", src, batch_size=7, timeout=15, column_mappings=["a"] ) @@ -394,7 +394,7 @@ def test_batch_size_timeout_accept_positional(self, mock_logger): module, pyc_cursor, _, _ = _mock_pycore() src = pa.table({"a": [1, 2]}) - with patch.dict("sys.modules", {"mssql_py_core": module}): + with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": module}): cur.bulkcopy_arrow("dbo.t", src, 7, 15) _, kwargs = pyc_cursor.bulkcopy_arrow.call_args @@ -407,7 +407,7 @@ def test_sensitive_fields_cleared_after_success(self, mock_logger): cur = _cursor_with_conn("Server=localhost;Database=d;UID=sa;PWD=secret") module, _, _, captured = _mock_pycore() - with patch.dict("sys.modules", {"mssql_py_core": module}): + with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": module}): cur.bulkcopy_arrow("t", pa.table({"a": [1]})) ctx = captured["ctx"] @@ -420,7 +420,7 @@ def test_resources_closed_on_success(self, mock_logger): cur = _cursor_with_conn("Server=localhost;Database=d;UID=sa;PWD=p") module, pyc_cursor, pyc_conn, _ = _mock_pycore() - with patch.dict("sys.modules", {"mssql_py_core": module}): + with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": module}): cur.bulkcopy_arrow("t", pa.table({"a": [1]})) pyc_cursor.close.assert_called_once() @@ -432,7 +432,7 @@ def test_core_exception_is_reraised_and_cleaned_up(self, mock_logger): cur = _cursor_with_conn("Server=localhost;Database=d;UID=sa;PWD=p") module, pyc_cursor, pyc_conn, captured = _mock_pycore(raise_exc=ValueError("boom")) - with patch.dict("sys.modules", {"mssql_py_core": module}): + with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": module}): with pytest.raises(ValueError, match="boom"): cur.bulkcopy_arrow("t", pa.table({"a": [1]})) @@ -449,7 +449,7 @@ def test_cleanup_swallows_close_errors(self, mock_logger): module, pyc_cursor, pyc_conn, _ = _mock_pycore() pyc_cursor.close.side_effect = RuntimeError("close failed") - with patch.dict("sys.modules", {"mssql_py_core": module}): + with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": module}): result = cur.bulkcopy_arrow("t", pa.table({"a": [1]})) # result is still returned; the close error was swallowed and logged