Repository navigation
FIX: Bound pooled connection checkout on unresponsive sockets #859
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,6 +6,7 @@ | |
| #include <chrono> | ||
| #include <exception> | ||
| #include <memory> | ||
| #include <limits> | ||
| #include <vector> | ||
|
|
||
| // Logging uses LOG() macro for all diagnostic output | ||
|
|
@@ -60,6 +61,22 @@ std::shared_ptr<Connection> ConnectionPool::acquire(const std::u16string& connSt | |
| const py::dict& attrs_before, | ||
| const py::object& token_factory) { | ||
| PERF_TIMER("ConnectionPool::acquire"); | ||
| // A login timeout does not cover SQLSetConnectAttr on an existing socket. | ||
| // Use it for bounded candidate reset as well; zero/unspecified | ||
| // retains a finite validation default rather than an OS TCP retry budget. | ||
| unsigned int validation_timeout = 15; | ||
| const py::int_ login_timeout_attr(SQL_ATTR_LOGIN_TIMEOUT); | ||
| if (attrs_before.contains(login_timeout_attr)) { | ||
| const auto requested = attrs_before[login_timeout_attr].cast<long long>(); | ||
| if (requested > 0) { | ||
| validation_timeout = static_cast<unsigned int>(std::min<long long>( | ||
| requested, std::numeric_limits<unsigned int>::max())); | ||
| } | ||
| } | ||
| const py::int_ autocommit_attr(SQL_ATTR_AUTOCOMMIT); | ||
| const bool autocommit = attrs_before.contains(autocommit_attr) | ||
| ? attrs_before[autocommit_attr].cast<int>() != SQL_AUTOCOMMIT_OFF | ||
| : true; | ||
| std::vector<std::shared_ptr<Connection>> to_disconnect; | ||
| std::shared_ptr<Connection> valid_conn = nullptr; | ||
| bool needs_connect = false; | ||
|
|
@@ -131,6 +148,7 @@ std::shared_ptr<Connection> ConnectionPool::acquire(const std::u16string& connSt | |
|
|
||
| // Validate the candidate outside the mutex. | ||
| bool reuse_candidate = false; | ||
| bool validation_attempted = false; | ||
| try { | ||
| if (token_factory && !token_factory.is_none() && | ||
| candidate->isTokenNearExpiry(TOKEN_EXPIRY_THRESHOLD_SECS)) { | ||
|
|
@@ -169,7 +187,10 @@ std::shared_ptr<Connection> ConnectionPool::acquire(const std::u16string& connSt | |
| // is taken in practice. We favor never handing out a token | ||
| // that may expire mid-query over avoiding the churn. | ||
| candidate->setTokenExpiry(fresh_expiry); | ||
| reuse_candidate = candidate->isAlive() && candidate->reset(); | ||
| if (candidate->isAlive()) { | ||
| validation_attempted = true; | ||
| reuse_candidate = candidate->reset(validation_timeout, autocommit); | ||
| } | ||
| if (!reuse_candidate) { | ||
| // The token is still valid but the socket is dead | ||
| // (isAlive()/reset() failed). We already minted the | ||
|
|
@@ -213,7 +234,10 @@ std::shared_ptr<Connection> ConnectionPool::acquire(const std::u16string& connSt | |
| } | ||
| } | ||
| } else { | ||
| reuse_candidate = candidate->isAlive() && candidate->reset(); | ||
| if (candidate->isAlive()) { | ||
| validation_attempted = true; | ||
| reuse_candidate = candidate->reset(validation_timeout, autocommit); | ||
| } | ||
| } | ||
| } catch (const std::exception& ex) { | ||
| LOG("Candidate connection validation failed: %s", ex.what()); | ||
|
|
@@ -232,10 +256,10 @@ std::shared_ptr<Connection> ConnectionPool::acquire(const std::u16string& connSt | |
| if (_current_size > 0) --_current_size; | ||
| } | ||
|
|
||
| // If a rotated token was captured, reserve a slot and reopen with it | ||
| // immediately instead of churning through the remaining candidates | ||
| // (which hold the same stale token and would all be discarded anyway). | ||
| if (have_pending_token) { | ||
| // After a reset failure, open a replacement instead of spending another | ||
| // timeout on each idle socket. Token rotation likewise opens directly | ||
| // using the captured token. | ||
| if (have_pending_token || validation_attempted) { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This intentionally opens a replacement after one failed validation, even when another healthy idle candidate exists. Please add a mixed stale/healthy pool test, including replacement-login failure, and verify that the remaining healthy connection stays reusable. This should make the intended availability-versus-latency trade-off explicit. |
||
| std::lock_guard<std::mutex> lock(_mutex); | ||
| if (_current_size < _max_size) { | ||
| // Reserve the slot here but construct the Connection outside | ||
|
|
@@ -246,11 +270,12 @@ std::shared_ptr<Connection> ConnectionPool::acquire(const std::u16string& connSt | |
| needs_connect = true; | ||
| break; | ||
| } | ||
| // Pool momentarily full; fall through and retry the loop. On the | ||
| // retry another near-expiry candidate may re-invoke the factory and | ||
| // overwrite pending_attrs/pending_expiry with a newer token. That | ||
| // needs a full pool AND a simultaneous rotation, is rare, and is | ||
| // harmless: we simply reopen with the most recently minted token. | ||
| if (validation_attempted) { | ||
| throw std::runtime_error( | ||
| "ConnectionPool::acquire: pool size limit reached"); | ||
| } | ||
| // A concurrent acquire reserved the slot freed by token rotation. | ||
| // Retry token comparison with another idle candidate. | ||
| } | ||
| } | ||
|
|
||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
High Priority: Please add fault-injection tests for timeout read/set/restore failures and autocommit configuration failure. Verify that a failed candidate is never returned, pool capacity remains correct, and subsequent checkout recovers. The successful restoration test does not cover these failure transitions.