FIX: Restore bulk copy after upgrading from 1.15.0

CHANGELOG.md

@@ -104,6 +104,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
   before; users should call `cursor.setinputsizes()` to work around this.
 
 ### Fixed
+- **GH-855:** Bulk copy and internal async queries load the Rust extension directly
+  when upgrading from 1.15.0 removes `mssql_py_core/__init__.py`. Existing affected
+  installations can restore the initializer with
+  `python -m pip install --force-reinstall --no-deps mssql-python-rs==0.3.0`,
+  followed by restarting Python.
 - **GH-754:** Pooled connections are now rolled back and restored to autocommit
   mode before being parked. This prevents an empty transaction from remaining
   visible on an idle SQL Server session after `Connection.close()`. Abandoned

mssql_python/async_query/_native.py

@@ -7,7 +7,8 @@
 def load_py_core() -> ModuleType:
     """Load the PyO3 extension that owns asynchronous TDS operations."""
     try:
-        py_core = import_module("mssql_py_core")
+        # Upgrading from a bundled core can remove the package's __init__.py.
+        py_core = import_module("mssql_py_core.mssql_py_core")
     except ImportError as exc:
         raise ImportError(
             "Async query support requires the mssql-python-rs dependency "

mssql_python/cursor.py

@@ -16,6 +16,7 @@
 import uuid
 import datetime
 import warnings
+from importlib import import_module
 from typing import List, Mapping, Union, Any, Optional, Tuple, Sequence, TYPE_CHECKING, Iterable
 from mssql_python.constants import ConstantsDDBC as ddbc_sql_const, SQLTypes
 from mssql_python.helpers import check_error, connstr_to_pycore_params
@@ -3447,9 +3448,10 @@ def _looks_like_arrow_source(self, data) -> bool:
     @staticmethod
     def _bulkcopy_core_and_validate(table_name, batch_size, timeout):
         """Import the native core and validate the args shared by ``bulkcopy``
-        and ``bulkcopy_arrow``. Returns the imported ``mssql_py_core`` module."""
+        and ``bulkcopy_arrow``. Returns the native ``mssql_py_core`` extension."""
         try:
-            import mssql_py_core
+            # Upgrading from a bundled core can remove the package's __init__.py.
+            mssql_py_core = import_module("mssql_py_core.mssql_py_core")
         except ImportError as exc:
             logger.error("bulkcopy: Failed to import mssql_py_core module")
             raise ImportError(

tests/AsyncTest/test_001_async_query_native.py

@@ -1,3 +1,5 @@
+import sys
+from importlib.machinery import ModuleSpec
 from types import ModuleType
 
 import pytest
@@ -19,15 +21,29 @@ def test_async_entry_points_are_internal():
 
 
 def test_load_py_core_uses_direct_native_dependency():
-    pytest.importorskip("mssql_py_core", exc_type=ImportError)
+    expected = pytest.importorskip("mssql_py_core.mssql_py_core", exc_type=ImportError)
 
     py_core = native.load_py_core()
-    assert py_core.__name__ == "mssql_py_core"
+    assert py_core is expected
     assert py_core.PyAsyncConnection.__name__ == "PyAsyncConnection"
     assert py_core.PyAsyncCursor.__name__ == "PyAsyncCursor"
     assert getattr(py_core, "PyAsyncCursor").__name__ == "PyAsyncCursor"
 
 
+def test_load_py_core_without_package_initializer(monkeypatch):
+    package = ModuleType("mssql_py_core")
+    package.__path__ = []
+    package.__spec__ = ModuleSpec("mssql_py_core", loader=None, is_package=True)
+    py_core = ModuleType("mssql_py_core.mssql_py_core")
+    py_core.PyAsyncConnection = type("PyAsyncConnection", (), {})
+    py_core.PyAsyncCursor = type("PyAsyncCursor", (), {})
+    monkeypatch.setitem(sys.modules, "mssql_py_core", package)
+    monkeypatch.setitem(sys.modules, "mssql_py_core.mssql_py_core", py_core)
+
+    assert not hasattr(package, "PyAsyncConnection")
+    assert native.load_py_core() is py_core
+
+
 def test_load_py_core_reports_missing_dependency(monkeypatch):
     def raise_import_error(name):
         raise ImportError(f"No module named {name}")
@@ -41,7 +57,7 @@ def raise_import_error(name):
 
 
 def test_load_py_core_reports_missing_async_types(monkeypatch):
-    py_core = ModuleType("mssql_py_core")
+    py_core = ModuleType("mssql_py_core.mssql_py_core")
     monkeypatch.setattr(native, "import_module", lambda name: py_core)
 
     with pytest.raises(ImportError, match="PyAsyncConnection, PyAsyncCursor"):

tests/test_008_auth.py

@@ -877,7 +877,7 @@ def fake_get_raw_token(auth_type, credential_kwargs=None):
         mock_pycore_module.PyCoreConnection = lambda ctx, **kwargs: mock_pycore_conn
 
         with (
-            patch.dict("sys.modules", {"mssql_py_core": mock_pycore_module}),
+            patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": mock_pycore_module}),
             patch("mssql_python.auth.AADAuth.get_raw_token", side_effect=fake_get_raw_token),
         ):
             cursor.bulkcopy("dbo.test_table", [(1, "row")], timeout=10)

tests/test_020_bulkcopy_auth_cleanup.py

@@ -10,7 +10,10 @@
 """
 
 import secrets
+import sys
 from enum import IntEnum
+from importlib.machinery import ModuleSpec
+from types import ModuleType
 from unittest.mock import MagicMock, patch
 
 import pytest
@@ -38,6 +41,42 @@ def _make_cursor(connection_str, auth_type):
     return cursor
 
 
+@pytest.mark.parametrize("method", ["bulkcopy", "bulkcopy_arrow"])
+def test_bulkcopy_without_package_initializer(monkeypatch, method):
+    package = ModuleType("mssql_py_core")
+    package.__path__ = []
+    package.__spec__ = ModuleSpec("mssql_py_core", loader=None, is_package=True)
+    native = ModuleType("mssql_py_core.mssql_py_core")
+    native.PyCoreConnection = MagicMock()
+    connection = native.PyCoreConnection.return_value
+    core_cursor = connection.cursor.return_value
+    expected = {"rows_copied": 1, "batch_count": 1, "elapsed_time": 0.1}
+    getattr(core_cursor, method).return_value = expected
+    monkeypatch.setitem(sys.modules, "mssql_py_core", package)
+    monkeypatch.setitem(sys.modules, "mssql_py_core.mssql_py_core", native)
+    cursor = _make_cursor("Server=example.test;Database=testdb;Trusted_Connection=yes", None)
+    source = [(1,)] if method == "bulkcopy" else object()
+
+    assert not hasattr(package, "PyCoreConnection")
+    assert getattr(cursor, method)("dbo.test_table", source) == expected
+    native.PyCoreConnection.assert_called_once()
+    getattr(core_cursor, method).assert_called_once()
+    core_cursor.close.assert_called_once()
+    connection.close.assert_called_once()
+
+
+@pytest.mark.parametrize("method", ["bulkcopy", "bulkcopy_arrow"])
+def test_bulkcopy_reports_missing_native_dependency(monkeypatch, method):
+    monkeypatch.setitem(sys.modules, "mssql_py_core.mssql_py_core", None)
+    cursor = _make_cursor("Server=example.test;Database=testdb;Trusted_Connection=yes", None)
+    source = [(1,)] if method == "bulkcopy" else object()
+
+    with pytest.raises(ImportError, match="Bulk copy requires") as exc_info:
+        getattr(cursor, method)("dbo.test_table", source)
+
+    assert isinstance(exc_info.value.__cause__, ImportError)
+
+
 class TestBulkcopyAuthCleanup:
     """Verify cursor.bulkcopy strips stale auth fields after token acquisition."""
 
@@ -71,7 +110,7 @@ def capture_context(ctx, **kwargs):
         mock_pycore_module.PyCoreConnection = capture_context
 
         with (
-            patch.dict("sys.modules", {"mssql_py_core": mock_pycore_module}),
+            patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": mock_pycore_module}),
             patch("mssql_python.auth.AADAuth.get_raw_token", return_value=SAMPLE_TOKEN),
         ):
             cursor.bulkcopy("dbo.test_table", [(1, "row")], timeout=10)
@@ -109,7 +148,7 @@ def capture_context(ctx, **kwargs):
         mock_pycore_module = MagicMock()
         mock_pycore_module.PyCoreConnection = capture_context
 
-        with patch.dict("sys.modules", {"mssql_py_core": mock_pycore_module}):
+        with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": mock_pycore_module}):
             cursor.bulkcopy("dbo.test_table", [(1, "row")], timeout=10)
 
         assert "access_token" not in captured_context
@@ -159,7 +198,7 @@ def capture_context(ctx, **kwargs):
         mock_pycore_module = MagicMock()
         mock_pycore_module.PyCoreConnection = capture_context
 
-        with patch.dict("sys.modules", {"mssql_py_core": mock_pycore_module}):
+        with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": mock_pycore_module}):
             cursor.bulkcopy("dbo.test_table", [(1, "row")], timeout=10)
 
         # The credential was consulted for a fresh token.
@@ -186,7 +225,7 @@ def test_token_provider_get_token_failure_rewrapped(self, mock_logger):
 
         mock_pycore_module = MagicMock()
 
-        with patch.dict("sys.modules", {"mssql_py_core": mock_pycore_module}):
+        with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": mock_pycore_module}):
             with pytest.raises(OperationalError) as exc_info:
                 cursor.bulkcopy("dbo.test_table", [(1, "row")], timeout=10)
 
@@ -219,7 +258,7 @@ def test_each_bulkcopy_reacquires_fresh_token(self, mock_logger):
         mock_pycore_module = MagicMock()
         mock_pycore_module.PyCoreConnection = lambda ctx, **kwargs: mock_pycore_conn
 
-        with patch.dict("sys.modules", {"mssql_py_core": mock_pycore_module}):
+        with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": mock_pycore_module}):
             cursor.bulkcopy("dbo.t", [(1, "a")], timeout=10)
             cursor.bulkcopy("dbo.t", [(2, "b")], timeout=10)
             cursor.bulkcopy("dbo.t", [(3, "c")], timeout=10)
@@ -255,7 +294,7 @@ def test_transient_failure_then_recovery(self, mock_logger):
         mock_pycore_module = MagicMock()
         mock_pycore_module.PyCoreConnection = lambda ctx, **kwargs: mock_pycore_conn
 
-        with patch.dict("sys.modules", {"mssql_py_core": mock_pycore_module}):
+        with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": mock_pycore_module}):
             with pytest.raises(OperationalError):
                 cursor.bulkcopy("dbo.t", [(1, "a")], timeout=10)
             # Cursor still works after the transient failure.
@@ -285,7 +324,7 @@ def test_empty_token_from_provider_raises_interface_error(self, mock_logger):
 
         mock_pycore_module = MagicMock()
 
-        with patch.dict("sys.modules", {"mssql_py_core": mock_pycore_module}):
+        with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": mock_pycore_module}):
             with pytest.raises(InterfaceError) as exc_info:
                 cursor.bulkcopy("dbo.test_table", [(1, "row")], timeout=10)
 
@@ -312,7 +351,7 @@ def capture_context(ctx, **kwargs):
     mock_pycore_module = MagicMock()
     mock_pycore_module.PyCoreConnection = capture_context
 
-    with patch.dict("sys.modules", {"mssql_py_core": mock_pycore_module}):
+    with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": mock_pycore_module}):
         cursor.bulkcopy("dbo.test_table", [(1, "row")], timeout=10)
 
     return captured_context

tests/test_024_bulkcopy_arrow.py

@@ -25,7 +25,7 @@
 # containers without the extension, or pyarrow absent). exc_type=ImportError
 # so a module that imports but fails its native extension is skipped, not
 # collected as an error (pytest >= 9.1 default only skips ModuleNotFoundError).
-mssql_py_core = pytest.importorskip("mssql_py_core", exc_type=ImportError)
+mssql_py_core = pytest.importorskip("mssql_py_core.mssql_py_core", exc_type=ImportError)
 pa = pytest.importorskip("pyarrow")
 
 LIVE_DB = pytest.mark.skipif(
@@ -219,7 +219,7 @@ def test_timeout_bool_rejected(self):
 
     def test_missing_pycore_raises_importerror(self):
         cur = _bare_cursor()
-        with patch.dict("sys.modules", {"mssql_py_core": None}):
+        with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": None}):
             with pytest.raises(ImportError, match="mssql_py_core"):
                 cur.bulkcopy_arrow("t", pa.table({"a": [1]}))
 
@@ -372,7 +372,7 @@ def test_success_returns_result_and_forwards_args(self, mock_logger):
         module, pyc_cursor, _, _ = _mock_pycore()
         src = pa.table({"a": [1, 2]})
 
-        with patch.dict("sys.modules", {"mssql_py_core": module}):
+        with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": module}):
             result = cur.bulkcopy_arrow(
                 "dbo.t", src, batch_size=7, timeout=15, column_mappings=["a"]
             )
@@ -394,7 +394,7 @@ def test_batch_size_timeout_accept_positional(self, mock_logger):
         module, pyc_cursor, _, _ = _mock_pycore()
         src = pa.table({"a": [1, 2]})
 
-        with patch.dict("sys.modules", {"mssql_py_core": module}):
+        with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": module}):
             cur.bulkcopy_arrow("dbo.t", src, 7, 15)
 
         _, kwargs = pyc_cursor.bulkcopy_arrow.call_args
@@ -407,7 +407,7 @@ def test_sensitive_fields_cleared_after_success(self, mock_logger):
         cur = _cursor_with_conn("Server=localhost;Database=d;UID=sa;PWD=secret")
         module, _, _, captured = _mock_pycore()
 
-        with patch.dict("sys.modules", {"mssql_py_core": module}):
+        with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": module}):
             cur.bulkcopy_arrow("t", pa.table({"a": [1]}))
 
         ctx = captured["ctx"]
@@ -420,7 +420,7 @@ def test_resources_closed_on_success(self, mock_logger):
         cur = _cursor_with_conn("Server=localhost;Database=d;UID=sa;PWD=p")
         module, pyc_cursor, pyc_conn, _ = _mock_pycore()
 
-        with patch.dict("sys.modules", {"mssql_py_core": module}):
+        with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": module}):
             cur.bulkcopy_arrow("t", pa.table({"a": [1]}))
 
         pyc_cursor.close.assert_called_once()
@@ -432,7 +432,7 @@ def test_core_exception_is_reraised_and_cleaned_up(self, mock_logger):
         cur = _cursor_with_conn("Server=localhost;Database=d;UID=sa;PWD=p")
         module, pyc_cursor, pyc_conn, captured = _mock_pycore(raise_exc=ValueError("boom"))
 
-        with patch.dict("sys.modules", {"mssql_py_core": module}):
+        with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": module}):
             with pytest.raises(ValueError, match="boom"):
                 cur.bulkcopy_arrow("t", pa.table({"a": [1]}))
 
@@ -449,7 +449,7 @@ def test_cleanup_swallows_close_errors(self, mock_logger):
         module, pyc_cursor, pyc_conn, _ = _mock_pycore()
         pyc_cursor.close.side_effect = RuntimeError("close failed")
 
-        with patch.dict("sys.modules", {"mssql_py_core": module}):
+        with patch.dict("sys.modules", {"mssql_py_core.mssql_py_core": module}):
             result = cur.bulkcopy_arrow("t", pa.table({"a": [1]}))
 
         # result is still returned; the close error was swallowed and logged